001package ca.uhn.fhir.jpa.dao; 002 003/*- 004 * #%L 005 * HAPI FHIR Storage api 006 * %% 007 * Copyright (C) 2014 - 2022 Smile CDR, Inc. 008 * %% 009 * Licensed under the Apache License, Version 2.0 (the "License"); 010 * you may not use this file except in compliance with the License. 011 * You may obtain a copy of the License at 012 * 013 * http://www.apache.org/licenses/LICENSE-2.0 014 * 015 * Unless required by applicable law or agreed to in writing, software 016 * distributed under the License is distributed on an "AS IS" BASIS, 017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 018 * See the License for the specific language governing permissions and 019 * limitations under the License. 020 * #L% 021 */ 022 023import ca.uhn.fhir.context.FhirContext; 024import ca.uhn.fhir.context.RuntimeResourceDefinition; 025import ca.uhn.fhir.i18n.Msg; 026import ca.uhn.fhir.interceptor.api.HookParams; 027import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster; 028import ca.uhn.fhir.interceptor.api.Pointcut; 029import ca.uhn.fhir.interceptor.model.RequestPartitionId; 030import ca.uhn.fhir.interceptor.model.TransactionWriteOperationsDetails; 031import ca.uhn.fhir.jpa.api.config.DaoConfig; 032import ca.uhn.fhir.jpa.api.dao.DaoRegistry; 033import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao; 034import ca.uhn.fhir.jpa.api.dao.IJpaDao; 035import ca.uhn.fhir.jpa.api.model.DaoMethodOutcome; 036import ca.uhn.fhir.jpa.api.model.DeleteConflict; 037import ca.uhn.fhir.jpa.api.model.DeleteConflictList; 038import ca.uhn.fhir.jpa.api.model.DeleteMethodOutcome; 039import ca.uhn.fhir.jpa.api.model.LazyDaoMethodOutcome; 040import ca.uhn.fhir.jpa.cache.IResourceVersionSvc; 041import ca.uhn.fhir.jpa.cache.ResourcePersistentIdMap; 042import ca.uhn.fhir.jpa.dao.tx.HapiTransactionService; 043import ca.uhn.fhir.jpa.delete.DeleteConflictUtil; 044import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource; 045import ca.uhn.fhir.jpa.model.entity.ModelConfig; 046import ca.uhn.fhir.jpa.model.entity.ResourceTable; 047import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage; 048import ca.uhn.fhir.jpa.searchparam.extractor.ResourceIndexedSearchParams; 049import ca.uhn.fhir.jpa.searchparam.matcher.InMemoryMatchResult; 050import ca.uhn.fhir.jpa.searchparam.matcher.InMemoryResourceMatcher; 051import ca.uhn.fhir.jpa.searchparam.matcher.SearchParamMatcher; 052import ca.uhn.fhir.model.api.IResource; 053import ca.uhn.fhir.model.api.ResourceMetadataKeyEnum; 054import ca.uhn.fhir.parser.DataFormatException; 055import ca.uhn.fhir.parser.IParser; 056import ca.uhn.fhir.rest.api.Constants; 057import ca.uhn.fhir.rest.api.PatchTypeEnum; 058import ca.uhn.fhir.rest.api.PreferReturnEnum; 059import ca.uhn.fhir.rest.api.RestOperationTypeEnum; 060import ca.uhn.fhir.rest.api.server.RequestDetails; 061import ca.uhn.fhir.rest.api.server.storage.DeferredInterceptorBroadcasts; 062import ca.uhn.fhir.rest.api.server.storage.TransactionDetails; 063import ca.uhn.fhir.rest.param.ParameterUtil; 064import ca.uhn.fhir.rest.server.RestfulServerUtils; 065import ca.uhn.fhir.rest.server.exceptions.BaseServerResponseException; 066import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; 067import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; 068import ca.uhn.fhir.rest.server.exceptions.MethodNotAllowedException; 069import ca.uhn.fhir.rest.server.exceptions.NotModifiedException; 070import ca.uhn.fhir.rest.server.exceptions.PayloadTooLargeException; 071import ca.uhn.fhir.rest.server.exceptions.PreconditionFailedException; 072import ca.uhn.fhir.rest.server.interceptor.IServerInterceptor; 073import ca.uhn.fhir.rest.server.method.BaseMethodBinding; 074import ca.uhn.fhir.rest.server.method.BaseResourceReturningMethodBinding; 075import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; 076import ca.uhn.fhir.rest.server.servlet.ServletSubRequestDetails; 077import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster; 078import ca.uhn.fhir.rest.server.util.ServletRequestUtil; 079import ca.uhn.fhir.util.AsyncUtil; 080import ca.uhn.fhir.util.ElementUtil; 081import ca.uhn.fhir.util.FhirTerser; 082import ca.uhn.fhir.util.ResourceReferenceInfo; 083import ca.uhn.fhir.util.StopWatch; 084import ca.uhn.fhir.util.ThreadPoolUtil; 085import ca.uhn.fhir.util.UrlUtil; 086import com.google.common.annotations.VisibleForTesting; 087import com.google.common.collect.ArrayListMultimap; 088import com.google.common.collect.ListMultimap; 089import org.apache.commons.lang3.StringUtils; 090import org.apache.commons.lang3.Validate; 091import org.hl7.fhir.dstu3.model.Bundle; 092import org.hl7.fhir.exceptions.FHIRException; 093import org.hl7.fhir.instance.model.api.IAnyResource; 094import org.hl7.fhir.instance.model.api.IBase; 095import org.hl7.fhir.instance.model.api.IBaseBinary; 096import org.hl7.fhir.instance.model.api.IBaseBundle; 097import org.hl7.fhir.instance.model.api.IBaseParameters; 098import org.hl7.fhir.instance.model.api.IBaseReference; 099import org.hl7.fhir.instance.model.api.IBaseResource; 100import org.hl7.fhir.instance.model.api.IIdType; 101import org.hl7.fhir.instance.model.api.IPrimitiveType; 102import org.hl7.fhir.r4.model.IdType; 103import org.slf4j.Logger; 104import org.slf4j.LoggerFactory; 105import org.springframework.beans.factory.annotation.Autowired; 106import org.springframework.core.task.SyncTaskExecutor; 107import org.springframework.core.task.TaskExecutor; 108import org.springframework.transaction.PlatformTransactionManager; 109import org.springframework.transaction.TransactionDefinition; 110import org.springframework.transaction.support.TransactionCallback; 111import org.springframework.transaction.support.TransactionTemplate; 112 113import javax.annotation.Nonnull; 114import javax.annotation.PostConstruct; 115import java.util.ArrayList; 116import java.util.Collection; 117import java.util.Comparator; 118import java.util.Date; 119import java.util.HashMap; 120import java.util.HashSet; 121import java.util.IdentityHashMap; 122import java.util.Iterator; 123import java.util.LinkedHashSet; 124import java.util.List; 125import java.util.Map; 126import java.util.Optional; 127import java.util.Set; 128import java.util.TreeSet; 129import java.util.concurrent.ConcurrentHashMap; 130import java.util.concurrent.CountDownLatch; 131import java.util.concurrent.TimeUnit; 132import java.util.regex.Pattern; 133import java.util.stream.Collectors; 134 135import static ca.uhn.fhir.util.StringUtil.toUtf8String; 136import static org.apache.commons.lang3.StringUtils.defaultString; 137import static org.apache.commons.lang3.StringUtils.isBlank; 138import static org.apache.commons.lang3.StringUtils.isNotBlank; 139 140public abstract class BaseTransactionProcessor { 141 142 public static final String URN_PREFIX = "urn:"; 143 public static final String URN_PREFIX_ESCAPED = UrlUtil.escapeUrlParam(URN_PREFIX); 144 public static final Pattern UNQUALIFIED_MATCH_URL_START = Pattern.compile("^[a-zA-Z0-9_]+="); 145 private static final Logger ourLog = LoggerFactory.getLogger(BaseTransactionProcessor.class); 146 public static final Pattern INVALID_PLACEHOLDER_PATTERN = Pattern.compile("[a-zA-Z]+:.*"); 147 private BaseStorageDao myDao; 148 @Autowired 149 private PlatformTransactionManager myTxManager; 150 @Autowired 151 private FhirContext myContext; 152 @Autowired 153 private ITransactionProcessorVersionAdapter myVersionAdapter; 154 @Autowired 155 private DaoRegistry myDaoRegistry; 156 @Autowired 157 private IInterceptorBroadcaster myInterceptorBroadcaster; 158 @Autowired 159 private HapiTransactionService myHapiTransactionService; 160 @Autowired 161 private DaoConfig myDaoConfig; 162 @Autowired 163 private ModelConfig myModelConfig; 164 @Autowired 165 private InMemoryResourceMatcher myInMemoryResourceMatcher; 166 @Autowired 167 private SearchParamMatcher mySearchParamMatcher; 168 169 private TaskExecutor myExecutor; 170 171 @Autowired 172 private IResourceVersionSvc myResourceVersionSvc; 173 174 @VisibleForTesting 175 public void setDaoConfig(DaoConfig theDaoConfig) { 176 myDaoConfig = theDaoConfig; 177 } 178 179 public ITransactionProcessorVersionAdapter getVersionAdapter() { 180 return myVersionAdapter; 181 } 182 183 @VisibleForTesting 184 public void setVersionAdapter(ITransactionProcessorVersionAdapter theVersionAdapter) { 185 myVersionAdapter = theVersionAdapter; 186 } 187 188 @PostConstruct 189 public void start() { 190 ourLog.trace("Starting transaction processor"); 191 } 192 193 private TaskExecutor getTaskExecutor() { 194 if (myExecutor == null) { 195 if (myDaoConfig.getBundleBatchPoolSize() > 1) { 196 myExecutor = ThreadPoolUtil.newThreadPool(myDaoConfig.getBundleBatchPoolSize(), myDaoConfig.getBundleBatchMaxPoolSize(), "bundle-batch-"); 197 } else { 198 SyncTaskExecutor executor = new SyncTaskExecutor(); 199 myExecutor = executor; 200 } 201 } 202 return myExecutor; 203 } 204 205 public <BUNDLE extends IBaseBundle> BUNDLE transaction(RequestDetails theRequestDetails, BUNDLE theRequest, boolean theNestedMode) { 206 if (theRequestDetails != null && theRequestDetails.getServer() != null && myDao != null) { 207 IServerInterceptor.ActionRequestDetails requestDetails = new IServerInterceptor.ActionRequestDetails(theRequestDetails, theRequest, "Bundle", null); 208 myDao.notifyInterceptors(RestOperationTypeEnum.TRANSACTION, requestDetails); 209 } 210 211 String actionName = "Transaction"; 212 IBaseBundle response = processTransactionAsSubRequest(theRequestDetails, theRequest, actionName, theNestedMode); 213 214 List<IBase> entries = myVersionAdapter.getEntries(response); 215 for (int i = 0; i < entries.size(); i++) { 216 if (ElementUtil.isEmpty(entries.get(i))) { 217 entries.remove(i); 218 i--; 219 } 220 } 221 222 return (BUNDLE) response; 223 } 224 225 public IBaseBundle collection(final RequestDetails theRequestDetails, IBaseBundle theRequest) { 226 String transactionType = myVersionAdapter.getBundleType(theRequest); 227 228 if (!org.hl7.fhir.r4.model.Bundle.BundleType.COLLECTION.toCode().equals(transactionType)) { 229 throw new InvalidRequestException(Msg.code(526) + "Can not process collection Bundle of type: " + transactionType); 230 } 231 232 ourLog.info("Beginning storing collection with {} resources", myVersionAdapter.getEntries(theRequest).size()); 233 234 TransactionTemplate txTemplate = new TransactionTemplate(myTxManager); 235 txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); 236 237 IBaseBundle resp = myVersionAdapter.createBundle(org.hl7.fhir.r4.model.Bundle.BundleType.BATCHRESPONSE.toCode()); 238 239 List<IBaseResource> resources = new ArrayList<>(); 240 for (final Object nextRequestEntry : myVersionAdapter.getEntries(theRequest)) { 241 IBaseResource resource = myVersionAdapter.getResource((IBase) nextRequestEntry); 242 resources.add(resource); 243 } 244 245 IBaseBundle transactionBundle = myVersionAdapter.createBundle("transaction"); 246 for (IBaseResource next : resources) { 247 IBase entry = myVersionAdapter.addEntry(transactionBundle); 248 myVersionAdapter.setResource(entry, next); 249 myVersionAdapter.setRequestVerb(entry, "PUT"); 250 myVersionAdapter.setRequestUrl(entry, next.getIdElement().toUnqualifiedVersionless().getValue()); 251 } 252 253 transaction(theRequestDetails, transactionBundle, false); 254 255 return resp; 256 } 257 258 private void populateEntryWithOperationOutcome(BaseServerResponseException caughtEx, IBase nextEntry) { 259 myVersionAdapter.populateEntryWithOperationOutcome(caughtEx, nextEntry); 260 } 261 262 private void handleTransactionCreateOrUpdateOutcome(IdSubstitutionMap idSubstitutions, Map<IIdType, DaoMethodOutcome> idToPersistedOutcome, 263 IIdType nextResourceId, DaoMethodOutcome outcome, 264 IBase newEntry, String theResourceType, 265 IBaseResource theRes, RequestDetails theRequestDetails) { 266 IIdType newId = outcome.getId().toUnqualified(); 267 IIdType resourceId = isPlaceholder(nextResourceId) ? nextResourceId : nextResourceId.toUnqualifiedVersionless(); 268 if (newId.equals(resourceId) == false) { 269 if (!nextResourceId.isEmpty()) { 270 idSubstitutions.put(resourceId, newId); 271 } 272 if (isPlaceholder(resourceId)) { 273 /* 274 * The correct way for substitution IDs to be is to be with no resource type, but we'll accept the qualified kind too just to be lenient. 275 */ 276 IIdType id = myContext.getVersion().newIdType(); 277 id.setValue(theResourceType + '/' + resourceId.getValue()); 278 idSubstitutions.put(id, newId); 279 } 280 } 281 282 populateIdToPersistedOutcomeMap(idToPersistedOutcome, newId, outcome); 283 284 if (outcome.getCreated()) { 285 myVersionAdapter.setResponseStatus(newEntry, toStatusString(Constants.STATUS_HTTP_201_CREATED)); 286 } else { 287 myVersionAdapter.setResponseStatus(newEntry, toStatusString(Constants.STATUS_HTTP_200_OK)); 288 } 289 Date lastModifier = getLastModified(theRes); 290 myVersionAdapter.setResponseLastModified(newEntry, lastModifier); 291 292 if (theRequestDetails != null) { 293 String prefer = theRequestDetails.getHeader(Constants.HEADER_PREFER); 294 PreferReturnEnum preferReturn = RestfulServerUtils.parsePreferHeader(null, prefer).getReturn(); 295 if (preferReturn != null) { 296 if (preferReturn == PreferReturnEnum.REPRESENTATION) { 297 if (outcome.getResource() != null) { 298 outcome.fireResourceViewCallbacks(); 299 myVersionAdapter.setResource(newEntry, outcome.getResource()); 300 } 301 } 302 } 303 } 304 305 } 306 307 /** 308 * Method which populates entry in idToPersistedOutcome. 309 * Will store whatever outcome is sent, unless the key already exists, then we only replace an instance if we find that the instance 310 * we are replacing with is non-lazy. This allows us to evaluate later more easily, as we _know_ we need access to these. 311 */ 312 private void populateIdToPersistedOutcomeMap(Map<IIdType, DaoMethodOutcome> idToPersistedOutcome, IIdType newId, DaoMethodOutcome outcome) { 313 //Prefer real method outcomes over lazy ones. 314 if (idToPersistedOutcome.containsKey(newId)) { 315 if (!(outcome instanceof LazyDaoMethodOutcome)) { 316 idToPersistedOutcome.put(newId, outcome); 317 } 318 } else { 319 idToPersistedOutcome.put(newId, outcome); 320 } 321 } 322 323 private Date getLastModified(IBaseResource theRes) { 324 return theRes.getMeta().getLastUpdated(); 325 } 326 327 public void setDao(BaseStorageDao theDao) { 328 myDao = theDao; 329 } 330 331 private IBaseBundle processTransactionAsSubRequest(RequestDetails theRequestDetails, IBaseBundle theRequest, String theActionName, boolean theNestedMode) { 332 BaseStorageDao.markRequestAsProcessingSubRequest(theRequestDetails); 333 try { 334 return processTransaction(theRequestDetails, theRequest, theActionName, theNestedMode); 335 } finally { 336 BaseStorageDao.clearRequestAsProcessingSubRequest(theRequestDetails); 337 } 338 } 339 340 @VisibleForTesting 341 public void setTxManager(PlatformTransactionManager theTxManager) { 342 myTxManager = theTxManager; 343 } 344 345 private IBaseBundle batch(final RequestDetails theRequestDetails, IBaseBundle theRequest, boolean theNestedMode) { 346 ourLog.info("Beginning batch with {} resources", myVersionAdapter.getEntries(theRequest).size()); 347 348 long start = System.currentTimeMillis(); 349 350 TransactionTemplate txTemplate = new TransactionTemplate(myTxManager); 351 txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); 352 353 IBaseBundle response = myVersionAdapter.createBundle(org.hl7.fhir.r4.model.Bundle.BundleType.BATCHRESPONSE.toCode()); 354 Map<Integer, Object> responseMap = new ConcurrentHashMap<>(); 355 356 List<IBase> requestEntries = myVersionAdapter.getEntries(theRequest); 357 int requestEntriesSize = requestEntries.size(); 358 359 // Now, run all non-gets sequentially, and all gets are submitted to the executor to run (potentially) in parallel 360 // The result is kept in the map to save the original position 361 List<RetriableBundleTask> getCalls = new ArrayList<>(); 362 List<RetriableBundleTask> nonGetCalls = new ArrayList<>(); 363 364 CountDownLatch completionLatch = new CountDownLatch(requestEntriesSize); 365 for (int i = 0; i < requestEntriesSize; i++) { 366 IBase nextRequestEntry = requestEntries.get(i); 367 RetriableBundleTask retriableBundleTask = new RetriableBundleTask(completionLatch, theRequestDetails, responseMap, i, nextRequestEntry, theNestedMode); 368 if (myVersionAdapter.getEntryRequestVerb(myContext, nextRequestEntry).equalsIgnoreCase("GET")) { 369 getCalls.add(retriableBundleTask); 370 } else { 371 nonGetCalls.add(retriableBundleTask); 372 } 373 } 374 //Execute all non-gets on calling thread. 375 nonGetCalls.forEach(RetriableBundleTask::run); 376 //Execute all gets (potentially in a pool) 377 getCalls.forEach(getCall -> getTaskExecutor().execute(getCall)); 378 379 // waiting for all async tasks to be completed 380 AsyncUtil.awaitLatchAndIgnoreInterrupt(completionLatch, 300L, TimeUnit.SECONDS); 381 382 // Now, create the bundle response in original order 383 Object nextResponseEntry; 384 for (int i = 0; i < requestEntriesSize; i++) { 385 386 nextResponseEntry = responseMap.get(i); 387 if (nextResponseEntry instanceof BaseServerResponseExceptionHolder) { 388 BaseServerResponseExceptionHolder caughtEx = (BaseServerResponseExceptionHolder) nextResponseEntry; 389 if (caughtEx.getException() != null) { 390 IBase nextEntry = myVersionAdapter.addEntry(response); 391 populateEntryWithOperationOutcome(caughtEx.getException(), nextEntry); 392 myVersionAdapter.setResponseStatus(nextEntry, toStatusString(caughtEx.getException().getStatusCode())); 393 } 394 } else { 395 myVersionAdapter.addEntry(response, (IBase) nextResponseEntry); 396 } 397 } 398 399 long delay = System.currentTimeMillis() - start; 400 ourLog.info("Batch completed in {}ms", delay); 401 402 return response; 403 } 404 405 @VisibleForTesting 406 public void setHapiTransactionService(HapiTransactionService theHapiTransactionService) { 407 myHapiTransactionService = theHapiTransactionService; 408 } 409 410 private IBaseBundle processTransaction(final RequestDetails theRequestDetails, final IBaseBundle theRequest, 411 final String theActionName, boolean theNestedMode) { 412 validateDependencies(); 413 414 String transactionType = myVersionAdapter.getBundleType(theRequest); 415 416 if (org.hl7.fhir.r4.model.Bundle.BundleType.BATCH.toCode().equals(transactionType)) { 417 return batch(theRequestDetails, theRequest, theNestedMode); 418 } 419 420 if (transactionType == null) { 421 String message = "Transaction Bundle did not specify valid Bundle.type, assuming " + Bundle.BundleType.TRANSACTION.toCode(); 422 ourLog.warn(message); 423 transactionType = org.hl7.fhir.r4.model.Bundle.BundleType.TRANSACTION.toCode(); 424 } 425 if (!org.hl7.fhir.r4.model.Bundle.BundleType.TRANSACTION.toCode().equals(transactionType)) { 426 throw new InvalidRequestException(Msg.code(527) + "Unable to process transaction where incoming Bundle.type = " + transactionType); 427 } 428 429 List<IBase> requestEntries = myVersionAdapter.getEntries(theRequest); 430 int numberOfEntries = requestEntries.size(); 431 432 if (myDaoConfig.getMaximumTransactionBundleSize() != null && numberOfEntries > myDaoConfig.getMaximumTransactionBundleSize()) { 433 throw new PayloadTooLargeException(Msg.code(528) + "Transaction Bundle Too large. Transaction bundle contains " + 434 numberOfEntries + 435 " which exceedes the maximum permitted transaction bundle size of " + myDaoConfig.getMaximumTransactionBundleSize()); 436 } 437 438 ourLog.debug("Beginning {} with {} resources", theActionName, numberOfEntries); 439 440 final TransactionDetails transactionDetails = new TransactionDetails(); 441 final StopWatch transactionStopWatch = new StopWatch(); 442 443 // Do all entries have a verb? 444 for (int i = 0; i < numberOfEntries; i++) { 445 IBase nextReqEntry = requestEntries.get(i); 446 String verb = myVersionAdapter.getEntryRequestVerb(myContext, nextReqEntry); 447 if (verb == null || !isValidVerb(verb)) { 448 throw new InvalidRequestException(Msg.code(529) + myContext.getLocalizer().getMessage(BaseStorageDao.class, "transactionEntryHasInvalidVerb", verb, i)); 449 } 450 } 451 452 /* 453 * We want to execute the transaction request bundle elements in the order 454 * specified by the FHIR specification (see TransactionSorter) so we save the 455 * original order in the request, then sort it. 456 * 457 * Entries with a type of GET are removed from the bundle so that they 458 * can be processed at the very end. We do this because the incoming resources 459 * are saved in a two-phase way in order to deal with interdependencies, and 460 * we want the GET processing to use the final indexing state 461 */ 462 final IBaseBundle response = myVersionAdapter.createBundle(org.hl7.fhir.r4.model.Bundle.BundleType.TRANSACTIONRESPONSE.toCode()); 463 List<IBase> getEntries = new ArrayList<>(); 464 final IdentityHashMap<IBase, Integer> originalRequestOrder = new IdentityHashMap<>(); 465 for (int i = 0; i < requestEntries.size(); i++) { 466 IBase requestEntry = requestEntries.get(i); 467 originalRequestOrder.put(requestEntry, i); 468 myVersionAdapter.addEntry(response); 469 if (myVersionAdapter.getEntryRequestVerb(myContext, requestEntry).equals("GET")) { 470 getEntries.add(requestEntry); 471 } 472 } 473 474 /* 475 * See FhirSystemDaoDstu3Test#testTransactionWithPlaceholderIdInMatchUrl 476 * Basically if the resource has a match URL that references a placeholder, 477 * we try to handle the resource with the placeholder first. 478 */ 479 Set<String> placeholderIds = new HashSet<>(); 480 for (IBase nextEntry : requestEntries) { 481 String fullUrl = myVersionAdapter.getFullUrl(nextEntry); 482 if (isNotBlank(fullUrl) && fullUrl.startsWith(URN_PREFIX)) { 483 placeholderIds.add(fullUrl); 484 } 485 } 486 requestEntries.sort(new TransactionSorter(placeholderIds)); 487 488 // perform all writes 489 prepareThenExecuteTransactionWriteOperations(theRequestDetails, theActionName, 490 transactionDetails, transactionStopWatch, 491 response, originalRequestOrder, requestEntries); 492 493 // perform all gets 494 // (we do these last so that the gets happen on the final state of the DB; 495 // see above note) 496 doTransactionReadOperations(theRequestDetails, response, 497 getEntries, originalRequestOrder, 498 transactionStopWatch, theNestedMode); 499 500 // Interceptor broadcast: JPA_PERFTRACE_INFO 501 if (CompositeInterceptorBroadcaster.hasHooks(Pointcut.JPA_PERFTRACE_INFO, myInterceptorBroadcaster, theRequestDetails)) { 502 String taskDurations = transactionStopWatch.formatTaskDurations(); 503 StorageProcessingMessage message = new StorageProcessingMessage(); 504 message.setMessage("Transaction timing:\n" + taskDurations); 505 HookParams params = new HookParams() 506 .add(RequestDetails.class, theRequestDetails) 507 .addIfMatchesType(ServletRequestDetails.class, theRequestDetails) 508 .add(StorageProcessingMessage.class, message); 509 CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequestDetails, Pointcut.JPA_PERFTRACE_INFO, params); 510 } 511 512 return response; 513 } 514 515 private void doTransactionReadOperations(final RequestDetails theRequestDetails, IBaseBundle theResponse, 516 List<IBase> theGetEntries, IdentityHashMap<IBase, Integer> theOriginalRequestOrder, 517 StopWatch theTransactionStopWatch, boolean theNestedMode) { 518 if (theGetEntries.size() > 0) { 519 theTransactionStopWatch.startTask("Process " + theGetEntries.size() + " GET entries"); 520 521 /* 522 * Loop through the request and process any entries of type GET 523 */ 524 for (IBase nextReqEntry : theGetEntries) { 525 if (theNestedMode) { 526 throw new InvalidRequestException(Msg.code(530) + "Can not invoke read operation on nested transaction"); 527 } 528 529 if (!(theRequestDetails instanceof ServletRequestDetails)) { 530 throw new MethodNotAllowedException(Msg.code(531) + "Can not call transaction GET methods from this context"); 531 } 532 533 ServletRequestDetails srd = (ServletRequestDetails) theRequestDetails; 534 Integer originalOrder = theOriginalRequestOrder.get(nextReqEntry); 535 IBase nextRespEntry = (IBase) myVersionAdapter.getEntries(theResponse).get(originalOrder); 536 537 ArrayListMultimap<String, String> paramValues = ArrayListMultimap.create(); 538 539 String transactionUrl = extractTransactionUrlOrThrowException(nextReqEntry, "GET"); 540 541 ServletSubRequestDetails requestDetails = ServletRequestUtil.getServletSubRequestDetails(srd, transactionUrl, paramValues); 542 543 String url = requestDetails.getRequestPath(); 544 545 BaseMethodBinding<?> method = srd.getServer().determineResourceMethod(requestDetails, url); 546 if (method == null) { 547 throw new IllegalArgumentException(Msg.code(532) + "Unable to handle GET " + url); 548 } 549 550 if (isNotBlank(myVersionAdapter.getEntryRequestIfMatch(nextReqEntry))) { 551 requestDetails.addHeader(Constants.HEADER_IF_MATCH, myVersionAdapter.getEntryRequestIfMatch(nextReqEntry)); 552 } 553 if (isNotBlank(myVersionAdapter.getEntryRequestIfNoneExist(nextReqEntry))) { 554 requestDetails.addHeader(Constants.HEADER_IF_NONE_EXIST, myVersionAdapter.getEntryRequestIfNoneExist(nextReqEntry)); 555 } 556 if (isNotBlank(myVersionAdapter.getEntryRequestIfNoneMatch(nextReqEntry))) { 557 requestDetails.addHeader(Constants.HEADER_IF_NONE_MATCH, myVersionAdapter.getEntryRequestIfNoneMatch(nextReqEntry)); 558 } 559 560 Validate.isTrue(method instanceof BaseResourceReturningMethodBinding, "Unable to handle GET {}", url); 561 try { 562 BaseResourceReturningMethodBinding methodBinding = (BaseResourceReturningMethodBinding) method; 563 requestDetails.setRestOperationType(methodBinding.getRestOperationType()); 564 565 IBaseResource resource = methodBinding.doInvokeServer(srd.getServer(), requestDetails); 566 if (paramValues.containsKey(Constants.PARAM_SUMMARY) || paramValues.containsKey(Constants.PARAM_CONTENT)) { 567 resource = filterNestedBundle(requestDetails, resource); 568 } 569 myVersionAdapter.setResource(nextRespEntry, resource); 570 myVersionAdapter.setResponseStatus(nextRespEntry, toStatusString(Constants.STATUS_HTTP_200_OK)); 571 } catch (NotModifiedException e) { 572 myVersionAdapter.setResponseStatus(nextRespEntry, toStatusString(Constants.STATUS_HTTP_304_NOT_MODIFIED)); 573 } catch (BaseServerResponseException e) { 574 ourLog.info("Failure processing transaction GET {}: {}", url, e.toString()); 575 myVersionAdapter.setResponseStatus(nextRespEntry, toStatusString(e.getStatusCode())); 576 populateEntryWithOperationOutcome(e, nextRespEntry); 577 } 578 } 579 theTransactionStopWatch.endCurrentTask(); 580 } 581 } 582 583 /** 584 * All of the write operations in the transaction (PUT, POST, etc.. basically anything 585 * except GET) are performed in their own database transaction before we do the reads. 586 * We do this because the reads (specifically the searches) often spawn their own 587 * secondary database transaction and if we allow that within the primary 588 * database transaction we can end up with deadlocks if the server is under 589 * heavy load with lots of concurrent transactions using all available 590 * database connections. 591 */ 592 private void prepareThenExecuteTransactionWriteOperations(RequestDetails theRequestDetails, String theActionName, 593 TransactionDetails theTransactionDetails, StopWatch theTransactionStopWatch, 594 IBaseBundle theResponse, IdentityHashMap<IBase, Integer> theOriginalRequestOrder, 595 List<IBase> theEntries) { 596 597 TransactionWriteOperationsDetails writeOperationsDetails = null; 598 if (haveWriteOperationsHooks(theRequestDetails)) { 599 writeOperationsDetails = buildWriteOperationsDetails(theEntries); 600 callWriteOperationsHook(Pointcut.STORAGE_TRANSACTION_WRITE_OPERATIONS_PRE, theRequestDetails, theTransactionDetails, writeOperationsDetails); 601 } 602 603 TransactionCallback<EntriesToProcessMap> txCallback = status -> { 604 final Set<IIdType> allIds = new LinkedHashSet<>(); 605 final IdSubstitutionMap idSubstitutions = new IdSubstitutionMap(); 606 final Map<IIdType, DaoMethodOutcome> idToPersistedOutcome = new HashMap<>(); 607 608 EntriesToProcessMap retVal = doTransactionWriteOperations(theRequestDetails, theActionName, 609 theTransactionDetails, allIds, 610 idSubstitutions, idToPersistedOutcome, 611 theResponse, theOriginalRequestOrder, 612 theEntries, theTransactionStopWatch); 613 614 theTransactionStopWatch.startTask("Commit writes to database"); 615 return retVal; 616 }; 617 EntriesToProcessMap entriesToProcess; 618 619 try { 620 entriesToProcess = myHapiTransactionService.execute(theRequestDetails, theTransactionDetails, txCallback); 621 } finally { 622 if (haveWriteOperationsHooks(theRequestDetails)) { 623 callWriteOperationsHook(Pointcut.STORAGE_TRANSACTION_WRITE_OPERATIONS_POST, theRequestDetails, theTransactionDetails, writeOperationsDetails); 624 } 625 } 626 627 theTransactionStopWatch.endCurrentTask(); 628 629 for (Map.Entry<IBase, IIdType> nextEntry : entriesToProcess.entrySet()) { 630 String responseLocation = nextEntry.getValue().toUnqualified().getValue(); 631 String responseEtag = nextEntry.getValue().getVersionIdPart(); 632 myVersionAdapter.setResponseLocation(nextEntry.getKey(), responseLocation); 633 myVersionAdapter.setResponseETag(nextEntry.getKey(), responseEtag); 634 } 635 } 636 637 private boolean haveWriteOperationsHooks(RequestDetails theRequestDetails) { 638 return CompositeInterceptorBroadcaster.hasHooks(Pointcut.STORAGE_TRANSACTION_WRITE_OPERATIONS_PRE, myInterceptorBroadcaster, theRequestDetails) || 639 CompositeInterceptorBroadcaster.hasHooks(Pointcut.STORAGE_TRANSACTION_WRITE_OPERATIONS_POST, myInterceptorBroadcaster, theRequestDetails); 640 } 641 642 private void callWriteOperationsHook(Pointcut thePointcut, RequestDetails theRequestDetails, TransactionDetails theTransactionDetails, TransactionWriteOperationsDetails theWriteOperationsDetails) { 643 HookParams params = new HookParams() 644 .add(TransactionDetails.class, theTransactionDetails) 645 .add(TransactionWriteOperationsDetails.class, theWriteOperationsDetails); 646 CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequestDetails, thePointcut, params); 647 } 648 649 private TransactionWriteOperationsDetails buildWriteOperationsDetails(List<IBase> theEntries) { 650 TransactionWriteOperationsDetails writeOperationsDetails; 651 List<String> updateRequestUrls = new ArrayList<>(); 652 List<String> conditionalCreateRequestUrls = new ArrayList<>(); 653 //Extract 654 for (IBase nextEntry : theEntries) { 655 String method = myVersionAdapter.getEntryRequestVerb(myContext, nextEntry); 656 if ("PUT".equals(method)) { 657 String requestUrl = myVersionAdapter.getEntryRequestUrl(nextEntry); 658 if (isNotBlank(requestUrl)) { 659 updateRequestUrls.add(requestUrl); 660 } 661 } else if ("POST".equals(method)) { 662 String requestUrl = myVersionAdapter.getEntryRequestIfNoneExist(nextEntry); 663 if (isNotBlank(requestUrl) && requestUrl.contains("?")) { 664 conditionalCreateRequestUrls.add(requestUrl); 665 } 666 } 667 } 668 669 writeOperationsDetails = new TransactionWriteOperationsDetails(); 670 writeOperationsDetails.setUpdateRequestUrls(updateRequestUrls); 671 writeOperationsDetails.setConditionalCreateRequestUrls(conditionalCreateRequestUrls); 672 return writeOperationsDetails; 673 } 674 675 private boolean isValidVerb(String theVerb) { 676 try { 677 return org.hl7.fhir.r4.model.Bundle.HTTPVerb.fromCode(theVerb) != null; 678 } catch (FHIRException theE) { 679 return false; 680 } 681 } 682 683 /** 684 * This method is called for nested bundles (e.g. if we received a transaction with an entry that 685 * was a GET search, this method is called on the bundle for the search result, that will be placed in the 686 * outer bundle). This method applies the _summary and _content parameters to the output of 687 * that bundle. 688 * <p> 689 * TODO: This isn't the most efficient way of doing this.. hopefully we can come up with something better in the future. 690 */ 691 private IBaseResource filterNestedBundle(RequestDetails theRequestDetails, IBaseResource theResource) { 692 IParser p = myContext.newJsonParser(); 693 RestfulServerUtils.configureResponseParser(theRequestDetails, p); 694 return p.parseResource(theResource.getClass(), p.encodeResourceToString(theResource)); 695 } 696 697 protected void validateDependencies() { 698 Validate.notNull(myContext); 699 Validate.notNull(myTxManager); 700 } 701 702 private IIdType newIdType(String theValue) { 703 return myContext.getVersion().newIdType().setValue(theValue); 704 } 705 706 @VisibleForTesting 707 public void setModelConfig(ModelConfig theModelConfig) { 708 myModelConfig = theModelConfig; 709 } 710 711 /** 712 * Searches for duplicate conditional creates and consolidates them. 713 */ 714 private void consolidateDuplicateConditionals(RequestDetails theRequestDetails, String theActionName, List<IBase> theEntries) { 715 final Set<String> keysWithNoFullUrl = new HashSet<>(); 716 final HashMap<String, String> keyToUuid = new HashMap<>(); 717 718 for (int index = 0, originalIndex = 0; index < theEntries.size(); index++, originalIndex++) { 719 IBase nextReqEntry = theEntries.get(index); 720 IBaseResource resource = myVersionAdapter.getResource(nextReqEntry); 721 if (resource != null) { 722 String verb = myVersionAdapter.getEntryRequestVerb(myContext, nextReqEntry); 723 String entryFullUrl = myVersionAdapter.getFullUrl(nextReqEntry); 724 String requestUrl = myVersionAdapter.getEntryRequestUrl(nextReqEntry); 725 String ifNoneExist = myVersionAdapter.getEntryRequestIfNoneExist(nextReqEntry); 726 727 // Conditional UPDATE 728 boolean consolidateEntryCandidate = false; 729 String conditionalUrl; 730 switch (verb) { 731 case "PUT": 732 conditionalUrl = requestUrl; 733 if (isNotBlank(requestUrl)) { 734 int questionMarkIndex = requestUrl.indexOf('?'); 735 if (questionMarkIndex >= 0 && requestUrl.length() > (questionMarkIndex + 1)) { 736 consolidateEntryCandidate = true; 737 } 738 } 739 break; 740 741 // Conditional CREATE 742 case "POST": 743 conditionalUrl = ifNoneExist; 744 if (isNotBlank(ifNoneExist)) { 745 if (isBlank(entryFullUrl) || !entryFullUrl.equals(requestUrl)) { 746 consolidateEntryCandidate = true; 747 } 748 } 749 break; 750 751 default: 752 continue; 753 } 754 755 if (isNotBlank(conditionalUrl) && !conditionalUrl.contains("?")) { 756 conditionalUrl = myContext.getResourceType(resource) + "?" + conditionalUrl; 757 } 758 759 String key = verb + "|" + conditionalUrl; 760 if (consolidateEntryCandidate) { 761 if (isBlank(entryFullUrl)) { 762 if (isNotBlank(conditionalUrl)) { 763 if (!keysWithNoFullUrl.add(key)) { 764 throw new InvalidRequestException( 765 Msg.code(2008) + "Unable to process " + theActionName + " - Request contains multiple anonymous entries (Bundle.entry.fullUrl not populated) with conditional URL: \"" + UrlUtil.sanitizeUrlPart(conditionalUrl) + "\". Does transaction request contain duplicates?"); 766 } 767 } 768 } else { 769 if (!keyToUuid.containsKey(key)) { 770 keyToUuid.put(key, entryFullUrl); 771 } else { 772 String msg = "Discarding transaction bundle entry " + originalIndex + " as it contained a duplicate conditional " + verb; 773 ourLog.info(msg); 774 // Interceptor broadcast: JPA_PERFTRACE_INFO 775 if (CompositeInterceptorBroadcaster.hasHooks(Pointcut.JPA_PERFTRACE_WARNING, myInterceptorBroadcaster, theRequestDetails)) { 776 StorageProcessingMessage message = new StorageProcessingMessage().setMessage(msg); 777 HookParams params = new HookParams() 778 .add(RequestDetails.class, theRequestDetails) 779 .addIfMatchesType(ServletRequestDetails.class, theRequestDetails) 780 .add(StorageProcessingMessage.class, message); 781 CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequestDetails, Pointcut.JPA_PERFTRACE_INFO, params); 782 } 783 784 theEntries.remove(index); 785 index--; 786 String existingUuid = keyToUuid.get(key); 787 replaceReferencesInEntriesWithConsolidatedUUID(theEntries, entryFullUrl, existingUuid); 788 } 789 } 790 } 791 } 792 } 793 } 794 795 /** 796 * Iterates over all entries, and if it finds any which have references which match the fullUrl of the entry that was consolidated out 797 * replace them with our new consolidated UUID 798 */ 799 private void replaceReferencesInEntriesWithConsolidatedUUID(List<IBase> theEntries, String theEntryFullUrl, String existingUuid) { 800 for (IBase nextEntry : theEntries) { 801 IBaseResource nextResource = myVersionAdapter.getResource(nextEntry); 802 for (IBaseReference nextReference : myContext.newTerser().getAllPopulatedChildElementsOfType(nextResource, IBaseReference.class)) { 803 // We're interested in any references directly to the placeholder ID, but also 804 // references that have a resource target that has the placeholder ID. 805 String nextReferenceId = nextReference.getReferenceElement().getValue(); 806 if (isBlank(nextReferenceId) && nextReference.getResource() != null) { 807 nextReferenceId = nextReference.getResource().getIdElement().getValue(); 808 } 809 if (theEntryFullUrl.equals(nextReferenceId)) { 810 nextReference.setReference(existingUuid); 811 nextReference.setResource(null); 812 } 813 } 814 } 815 } 816 817 /** 818 * Retrieves the next resource id (IIdType) from the base resource and next request entry. 819 * 820 * @param theBaseResource - base resource 821 * @param theNextReqEntry - next request entry 822 * @param theAllIds - set of all IIdType values 823 * @return 824 */ 825 private IIdType getNextResourceIdFromBaseResource(IBaseResource theBaseResource, 826 IBase theNextReqEntry, 827 Set<IIdType> theAllIds) { 828 IIdType nextResourceId = null; 829 if (theBaseResource != null) { 830 nextResourceId = theBaseResource.getIdElement(); 831 832 String fullUrl = myVersionAdapter.getFullUrl(theNextReqEntry); 833 if (isNotBlank(fullUrl)) { 834 IIdType fullUrlIdType = newIdType(fullUrl); 835 if (isPlaceholder(fullUrlIdType)) { 836 nextResourceId = fullUrlIdType; 837 } else if (!nextResourceId.hasIdPart()) { 838 nextResourceId = fullUrlIdType; 839 } 840 } 841 842 if (nextResourceId.hasIdPart() && !isPlaceholder(nextResourceId)) { 843 int colonIndex = nextResourceId.getIdPart().indexOf(':'); 844 if (colonIndex != -1) { 845 if (INVALID_PLACEHOLDER_PATTERN.matcher(nextResourceId.getIdPart()).matches()) { 846 throw new InvalidRequestException(Msg.code(533) + "Invalid placeholder ID found: " + nextResourceId.getIdPart() + " - Must be of the form 'urn:uuid:[uuid]' or 'urn:oid:[oid]'"); 847 } 848 } 849 } 850 851 if (nextResourceId.hasIdPart() && !nextResourceId.hasResourceType() && !isPlaceholder(nextResourceId)) { 852 nextResourceId = newIdType(toResourceName(theBaseResource.getClass()), nextResourceId.getIdPart()); 853 theBaseResource.setId(nextResourceId); 854 } 855 856 /* 857 * Ensure that the bundle doesn't have any duplicates, since this causes all kinds of weirdness 858 */ 859 if (isPlaceholder(nextResourceId)) { 860 if (!theAllIds.add(nextResourceId)) { 861 throw new InvalidRequestException(Msg.code(534) + myContext.getLocalizer().getMessage(BaseStorageDao.class, "transactionContainsMultipleWithDuplicateId", nextResourceId)); 862 } 863 } else if (nextResourceId.hasResourceType() && nextResourceId.hasIdPart()) { 864 IIdType nextId = nextResourceId.toUnqualifiedVersionless(); 865 if (!theAllIds.add(nextId)) { 866 throw new InvalidRequestException(Msg.code(535) + myContext.getLocalizer().getMessage(BaseStorageDao.class, "transactionContainsMultipleWithDuplicateId", nextId)); 867 } 868 } 869 870 } 871 872 return nextResourceId; 873 } 874 875 /** 876 * After pre-hooks have been called 877 */ 878 protected EntriesToProcessMap doTransactionWriteOperations(final RequestDetails theRequest, String theActionName, 879 TransactionDetails theTransactionDetails, Set<IIdType> theAllIds, 880 IdSubstitutionMap theIdSubstitutions, Map<IIdType, DaoMethodOutcome> theIdToPersistedOutcome, 881 IBaseBundle theResponse, IdentityHashMap<IBase, Integer> theOriginalRequestOrder, 882 List<IBase> theEntries, StopWatch theTransactionStopWatch) { 883 884 // During a transaction, we don't execute hooks, instead, we execute them all post-transaction. 885 theTransactionDetails.beginAcceptingDeferredInterceptorBroadcasts( 886 Pointcut.STORAGE_PRECOMMIT_RESOURCE_CREATED, 887 Pointcut.STORAGE_PRECOMMIT_RESOURCE_UPDATED, 888 Pointcut.STORAGE_PRECOMMIT_RESOURCE_DELETED 889 ); 890 try { 891 Set<String> deletedResources = new HashSet<>(); 892 DeleteConflictList deleteConflicts = new DeleteConflictList(); 893 EntriesToProcessMap entriesToProcess = new EntriesToProcessMap(); 894 Set<IIdType> nonUpdatedEntities = new HashSet<>(); 895 Set<IBasePersistedResource> updatedEntities = new HashSet<>(); 896 Map<String, IIdType> conditionalUrlToIdMap = new HashMap<>(); 897 List<IBaseResource> updatedResources = new ArrayList<>(); 898 Map<String, Class<? extends IBaseResource>> conditionalRequestUrls = new HashMap<>(); 899 900 /* 901 * Look for duplicate conditional creates and consolidate them 902 */ 903 consolidateDuplicateConditionals(theRequest, theActionName, theEntries); 904 905 /* 906 * Loop through the request and process any entries of type 907 * PUT, POST or DELETE 908 */ 909 for (int i = 0; i < theEntries.size(); i++) { 910 if (i % 250 == 0) { 911 ourLog.debug("Processed {} non-GET entries out of {} in transaction", i, theEntries.size()); 912 } 913 914 IBase nextReqEntry = theEntries.get(i); 915 IBaseResource res = myVersionAdapter.getResource(nextReqEntry); 916 IIdType nextResourceId = getNextResourceIdFromBaseResource(res, nextReqEntry, theAllIds); 917 918 String verb = myVersionAdapter.getEntryRequestVerb(myContext, nextReqEntry); 919 String resourceType = res != null ? myContext.getResourceType(res) : null; 920 Integer order = theOriginalRequestOrder.get(nextReqEntry); 921 IBase nextRespEntry = (IBase) myVersionAdapter.getEntries(theResponse).get(order); 922 923 theTransactionStopWatch.startTask("Bundle.entry[" + i + "]: " + verb + " " + defaultString(resourceType)); 924 925 switch (verb) { 926 case "POST": { 927 // CREATE 928 /* 929 * To preserve existing functionality, 930 * we will only verify that the request url is 931 * valid if it's provided at all. 932 * Otherwise, we'll ignore it 933 */ 934 String url = myVersionAdapter.getEntryRequestUrl(nextReqEntry); 935 if (isNotBlank(url)) { 936 extractAndVerifyTransactionUrlForEntry(nextReqEntry, verb); 937 } 938 validateResourcePresent(res, order, verb); 939 @SuppressWarnings("rawtypes") 940 IFhirResourceDao resourceDao = getDaoOrThrowException(res.getClass()); 941 res.setId((String) null); 942 DaoMethodOutcome outcome; 943 String matchUrl = myVersionAdapter.getEntryRequestIfNoneExist(nextReqEntry); 944 matchUrl = performIdSubstitutionsInMatchUrl(theIdSubstitutions, matchUrl); 945 outcome = resourceDao.create(res, matchUrl, false, theTransactionDetails, theRequest); 946 setConditionalUrlToBeValidatedLater(conditionalUrlToIdMap, matchUrl, outcome.getId()); 947 res.setId(outcome.getId()); 948 if (nextResourceId != null) { 949 handleTransactionCreateOrUpdateOutcome(theIdSubstitutions, theIdToPersistedOutcome, nextResourceId, outcome, nextRespEntry, resourceType, res, theRequest); 950 } 951 entriesToProcess.put(nextRespEntry, outcome.getId()); 952 if (outcome.getCreated() == false) { 953 nonUpdatedEntities.add(outcome.getId()); 954 } else { 955 if (isNotBlank(matchUrl)) { 956 conditionalRequestUrls.put(matchUrl, res.getClass()); 957 } 958 } 959 960 break; 961 } 962 case "DELETE": { 963 // DELETE 964 String url = extractAndVerifyTransactionUrlForEntry(nextReqEntry, verb); 965 UrlUtil.UrlParts parts = UrlUtil.parseUrl(url); 966 IFhirResourceDao<? extends IBaseResource> dao = toDao(parts, verb, url); 967 int status = Constants.STATUS_HTTP_204_NO_CONTENT; 968 if (parts.getResourceId() != null) { 969 IIdType deleteId = newIdType(parts.getResourceType(), parts.getResourceId()); 970 if (!deletedResources.contains(deleteId.getValueAsString())) { 971 DaoMethodOutcome outcome = dao.delete(deleteId, deleteConflicts, theRequest, theTransactionDetails); 972 if (outcome.getEntity() != null) { 973 deletedResources.add(deleteId.getValueAsString()); 974 entriesToProcess.put(nextRespEntry, outcome.getId()); 975 } 976 } 977 } else { 978 String matchUrl = parts.getResourceType() + '?' + parts.getParams(); 979 matchUrl = performIdSubstitutionsInMatchUrl(theIdSubstitutions, matchUrl); 980 DeleteMethodOutcome deleteOutcome = dao.deleteByUrl(matchUrl, deleteConflicts, theRequest); 981 setConditionalUrlToBeValidatedLater(conditionalUrlToIdMap, matchUrl, deleteOutcome.getId()); 982 List<ResourceTable> allDeleted = deleteOutcome.getDeletedEntities(); 983 for (ResourceTable deleted : allDeleted) { 984 deletedResources.add(deleted.getIdDt().toUnqualifiedVersionless().getValueAsString()); 985 } 986 if (allDeleted.isEmpty()) { 987 status = Constants.STATUS_HTTP_204_NO_CONTENT; 988 } 989 990 myVersionAdapter.setResponseOutcome(nextRespEntry, deleteOutcome.getOperationOutcome()); 991 } 992 993 myVersionAdapter.setResponseStatus(nextRespEntry, toStatusString(status)); 994 995 break; 996 } 997 case "PUT": { 998 // UPDATE 999 validateResourcePresent(res, order, verb); 1000 @SuppressWarnings("rawtypes") 1001 IFhirResourceDao resourceDao = getDaoOrThrowException(res.getClass()); 1002 1003 String url = extractAndVerifyTransactionUrlForEntry(nextReqEntry, verb); 1004 1005 DaoMethodOutcome outcome; 1006 UrlUtil.UrlParts parts = UrlUtil.parseUrl(url); 1007 if (isNotBlank(parts.getResourceId())) { 1008 String version = null; 1009 if (isNotBlank(myVersionAdapter.getEntryRequestIfMatch(nextReqEntry))) { 1010 version = ParameterUtil.parseETagValue(myVersionAdapter.getEntryRequestIfMatch(nextReqEntry)); 1011 } 1012 res.setId(newIdType(parts.getResourceType(), parts.getResourceId(), version)); 1013 outcome = resourceDao.update(res, null, false, false, theRequest, theTransactionDetails); 1014 } else { 1015 res.setId((String) null); 1016 String matchUrl; 1017 if (isNotBlank(parts.getParams())) { 1018 matchUrl = parts.getResourceType() + '?' + parts.getParams(); 1019 } else { 1020 matchUrl = parts.getResourceType(); 1021 } 1022 matchUrl = performIdSubstitutionsInMatchUrl(theIdSubstitutions, matchUrl); 1023 outcome = resourceDao.update(res, matchUrl, false, false, theRequest, theTransactionDetails); 1024 setConditionalUrlToBeValidatedLater(conditionalUrlToIdMap, matchUrl, outcome.getId()); 1025 if (Boolean.TRUE.equals(outcome.getCreated())) { 1026 conditionalRequestUrls.put(matchUrl, res.getClass()); 1027 } 1028 } 1029 1030 if (outcome.getCreated() == Boolean.FALSE 1031 || (outcome.getCreated() == Boolean.TRUE && outcome.getId().getVersionIdPartAsLong() > 1)) { 1032 updatedEntities.add(outcome.getEntity()); 1033 if (outcome.getResource() != null) { 1034 updatedResources.add(outcome.getResource()); 1035 } 1036 } 1037 1038 handleTransactionCreateOrUpdateOutcome(theIdSubstitutions, theIdToPersistedOutcome, nextResourceId, 1039 outcome, nextRespEntry, resourceType, res, theRequest); 1040 entriesToProcess.put(nextRespEntry, outcome.getId()); 1041 break; 1042 } 1043 case "PATCH": { 1044 // PATCH 1045 validateResourcePresent(res, order, verb); 1046 1047 String url = extractAndVerifyTransactionUrlForEntry(nextReqEntry, verb); 1048 UrlUtil.UrlParts parts = UrlUtil.parseUrl(url); 1049 1050 String matchUrl = toMatchUrl(nextReqEntry); 1051 matchUrl = performIdSubstitutionsInMatchUrl(theIdSubstitutions, matchUrl); 1052 String patchBody = null; 1053 String contentType; 1054 IBaseParameters patchBodyParameters = null; 1055 PatchTypeEnum patchType = null; 1056 1057 if (res instanceof IBaseBinary) { 1058 IBaseBinary binary = (IBaseBinary) res; 1059 if (binary.getContent() != null && binary.getContent().length > 0) { 1060 patchBody = toUtf8String(binary.getContent()); 1061 } 1062 contentType = binary.getContentType(); 1063 patchType = PatchTypeEnum.forContentTypeOrThrowInvalidRequestException(myContext, contentType); 1064 if (patchType == PatchTypeEnum.FHIR_PATCH_JSON || patchType == PatchTypeEnum.FHIR_PATCH_XML) { 1065 String msg = myContext.getLocalizer().getMessage(BaseTransactionProcessor.class, "fhirPatchShouldNotUseBinaryResource"); 1066 throw new InvalidRequestException(Msg.code(536) + msg); 1067 } 1068 } else if (res instanceof IBaseParameters) { 1069 patchBodyParameters = (IBaseParameters) res; 1070 patchType = PatchTypeEnum.FHIR_PATCH_JSON; 1071 } 1072 1073 if (patchBodyParameters == null) { 1074 if (isBlank(patchBody)) { 1075 String msg = myContext.getLocalizer().getMessage(BaseTransactionProcessor.class, "missingPatchBody"); 1076 throw new InvalidRequestException(Msg.code(537) + msg); 1077 } 1078 } 1079 1080 IFhirResourceDao<? extends IBaseResource> dao = toDao(parts, verb, url); 1081 IIdType patchId = myContext.getVersion().newIdType().setValue(parts.getResourceId()); 1082 DaoMethodOutcome outcome = dao.patch(patchId, matchUrl, patchType, patchBody, patchBodyParameters, theRequest); 1083 setConditionalUrlToBeValidatedLater(conditionalUrlToIdMap, matchUrl, outcome.getId()); 1084 updatedEntities.add(outcome.getEntity()); 1085 if (outcome.getResource() != null) { 1086 updatedResources.add(outcome.getResource()); 1087 } 1088 1089 break; 1090 } 1091 case "GET": 1092 break; 1093 default: 1094 throw new InvalidRequestException(Msg.code(538) + "Unable to handle verb in transaction: " + verb); 1095 1096 } 1097 1098 theTransactionStopWatch.endCurrentTask(); 1099 } 1100 1101 /* 1102 * Make sure that there are no conflicts from deletions. E.g. we can't delete something 1103 * if something else has a reference to it.. Unless the thing that has a reference to it 1104 * was also deleted as a part of this transaction, which is why we check this now at the 1105 * end. 1106 */ 1107 checkForDeleteConflicts(deleteConflicts, deletedResources, updatedResources); 1108 1109 theIdToPersistedOutcome.entrySet().forEach(idAndOutcome -> { 1110 theTransactionDetails.addResolvedResourceId(idAndOutcome.getKey(), idAndOutcome.getValue().getPersistentId()); 1111 }); 1112 1113 /* 1114 * Perform ID substitutions and then index each resource we have saved 1115 */ 1116 1117 resolveReferencesThenSaveAndIndexResources(theRequest, theTransactionDetails, 1118 theIdSubstitutions, theIdToPersistedOutcome, 1119 theTransactionStopWatch, entriesToProcess, 1120 nonUpdatedEntities, updatedEntities); 1121 1122 theTransactionStopWatch.endCurrentTask(); 1123 1124 // flush writes to db 1125 theTransactionStopWatch.startTask("Flush writes to database"); 1126 1127 flushSession(theIdToPersistedOutcome); 1128 1129 theTransactionStopWatch.endCurrentTask(); 1130 1131 /* 1132 * Double check we didn't allow any duplicates we shouldn't have 1133 */ 1134 if (conditionalRequestUrls.size() > 0) { 1135 theTransactionStopWatch.startTask("Check for conflicts in conditional resources"); 1136 } 1137 if (!myDaoConfig.isMassIngestionMode()) { 1138 validateNoDuplicates(theRequest, theActionName, conditionalRequestUrls, theIdToPersistedOutcome.values()); 1139 } 1140 1141 theTransactionStopWatch.endCurrentTask(); 1142 if (conditionalUrlToIdMap.size() > 0) { 1143 theTransactionStopWatch.startTask("Check that all conditionally created/updated entities actually match their conditionals."); 1144 } 1145 1146 if (!myDaoConfig.isMassIngestionMode()) { 1147 validateAllInsertsMatchTheirConditionalUrls(theIdToPersistedOutcome, conditionalUrlToIdMap, theRequest); 1148 } 1149 theTransactionStopWatch.endCurrentTask(); 1150 1151 for (IIdType next : theAllIds) { 1152 IIdType replacement = theIdSubstitutions.getForSource(next); 1153 if (replacement != null && !replacement.equals(next)) { 1154 ourLog.debug("Placeholder resource ID \"{}\" was replaced with permanent ID \"{}\"", next, replacement); 1155 } 1156 } 1157 1158 ListMultimap<Pointcut, HookParams> deferredBroadcastEvents = theTransactionDetails.endAcceptingDeferredInterceptorBroadcasts(); 1159 for (Map.Entry<Pointcut, HookParams> nextEntry : deferredBroadcastEvents.entries()) { 1160 Pointcut nextPointcut = nextEntry.getKey(); 1161 HookParams nextParams = nextEntry.getValue(); 1162 CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, nextPointcut, nextParams); 1163 } 1164 1165 DeferredInterceptorBroadcasts deferredInterceptorBroadcasts = new DeferredInterceptorBroadcasts(deferredBroadcastEvents); 1166 HookParams params = new HookParams() 1167 .add(RequestDetails.class, theRequest) 1168 .addIfMatchesType(ServletRequestDetails.class, theRequest) 1169 .add(DeferredInterceptorBroadcasts.class, deferredInterceptorBroadcasts) 1170 .add(TransactionDetails.class, theTransactionDetails) 1171 .add(IBaseBundle.class, theResponse); 1172 CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, Pointcut.STORAGE_TRANSACTION_PROCESSED, params); 1173 1174 theTransactionDetails.deferredBroadcastProcessingFinished(); 1175 1176 //finishedCallingDeferredInterceptorBroadcasts 1177 1178 return entriesToProcess; 1179 1180 } finally { 1181 if (theTransactionDetails.isAcceptingDeferredInterceptorBroadcasts()) { 1182 theTransactionDetails.endAcceptingDeferredInterceptorBroadcasts(); 1183 } 1184 } 1185 } 1186 1187 private void setConditionalUrlToBeValidatedLater(Map<String, IIdType> theConditionalUrlToIdMap, String theMatchUrl, IIdType theId) { 1188 if (!StringUtils.isBlank(theMatchUrl)) { 1189 theConditionalUrlToIdMap.put(theMatchUrl, theId); 1190 } 1191 } 1192 1193 /** 1194 * After transaction processing and resolution of indexes and references, we want to validate that the resources that were stored _actually_ 1195 * match the conditional URLs that they were brought in on. 1196 */ 1197 private void validateAllInsertsMatchTheirConditionalUrls(Map<IIdType, DaoMethodOutcome> theIdToPersistedOutcome, Map<String, IIdType> conditionalUrlToIdMap, RequestDetails theRequest) { 1198 conditionalUrlToIdMap.entrySet().stream() 1199 .filter(entry -> entry.getKey() != null) 1200 .forEach(entry -> { 1201 String matchUrl = entry.getKey(); 1202 IIdType value = entry.getValue(); 1203 DaoMethodOutcome daoMethodOutcome = theIdToPersistedOutcome.get(value); 1204 if (daoMethodOutcome != null && !daoMethodOutcome.isNop() && daoMethodOutcome.getResource() != null) { 1205 InMemoryMatchResult match = mySearchParamMatcher.match(matchUrl, daoMethodOutcome.getResource(), theRequest); 1206 if (ourLog.isDebugEnabled()) { 1207 ourLog.debug("Checking conditional URL [{}] against resource with ID [{}]: Supported?:[{}], Matched?:[{}]", matchUrl, value, match.supported(), match.matched()); 1208 } 1209 if (match.supported()) { 1210 if (!match.matched()) { 1211 throw new PreconditionFailedException(Msg.code(539) + "Invalid conditional URL \"" + matchUrl + "\". The given resource is not matched by this URL."); 1212 } 1213 ; 1214 } 1215 } 1216 }); 1217 } 1218 1219 /** 1220 * Checks for any delete conflicts. 1221 * 1222 * @param theDeleteConflicts - set of delete conflicts 1223 * @param theDeletedResources - set of deleted resources 1224 * @param theUpdatedResources - list of updated resources 1225 */ 1226 private void checkForDeleteConflicts(DeleteConflictList theDeleteConflicts, 1227 Set<String> theDeletedResources, 1228 List<IBaseResource> theUpdatedResources) { 1229 for (Iterator<DeleteConflict> iter = theDeleteConflicts.iterator(); iter.hasNext(); ) { 1230 DeleteConflict nextDeleteConflict = iter.next(); 1231 1232 /* 1233 * If we have a conflict, it means we can't delete Resource/A because 1234 * Resource/B has a reference to it. We'll ignore that conflict though 1235 * if it turns out we're also deleting Resource/B in this transaction. 1236 */ 1237 if (theDeletedResources.contains(nextDeleteConflict.getSourceId().toUnqualifiedVersionless().getValue())) { 1238 iter.remove(); 1239 continue; 1240 } 1241 1242 /* 1243 * And then, this is kind of a last ditch check. It's also ok to delete 1244 * Resource/A if Resource/B isn't being deleted, but it is being UPDATED 1245 * in this transaction, and the updated version of it has no references 1246 * to Resource/A any more. 1247 */ 1248 String sourceId = nextDeleteConflict.getSourceId().toUnqualifiedVersionless().getValue(); 1249 String targetId = nextDeleteConflict.getTargetId().toUnqualifiedVersionless().getValue(); 1250 Optional<IBaseResource> updatedSource = theUpdatedResources 1251 .stream() 1252 .filter(t -> sourceId.equals(t.getIdElement().toUnqualifiedVersionless().getValue())) 1253 .findFirst(); 1254 if (updatedSource.isPresent()) { 1255 List<ResourceReferenceInfo> referencesInSource = myContext.newTerser().getAllResourceReferences(updatedSource.get()); 1256 boolean sourceStillReferencesTarget = referencesInSource 1257 .stream() 1258 .anyMatch(t -> targetId.equals(t.getResourceReference().getReferenceElement().toUnqualifiedVersionless().getValue())); 1259 if (!sourceStillReferencesTarget) { 1260 iter.remove(); 1261 } 1262 } 1263 } 1264 DeleteConflictUtil.validateDeleteConflictsEmptyOrThrowException(myContext, theDeleteConflicts); 1265 } 1266 1267 /** 1268 * This method replaces any placeholder references in the 1269 * source transaction Bundle with their actual targets, then stores the resource contents and indexes 1270 * in the database. This is trickier than you'd think because of a couple of possibilities during the 1271 * save: 1272 * * There may be resources that have not changed (e.g. an update/PUT with a resource body identical 1273 * to what is already in the database) 1274 * * There may be resources with auto-versioned references, meaning we're replacing certain references 1275 * in the resource with a versioned references, referencing the current version at the time of the 1276 * transaction processing 1277 * * There may by auto-versioned references pointing to these unchanged targets 1278 * <p> 1279 * If we're not doing any auto-versioned references, we'll just iterate through all resources in the 1280 * transaction and save them one at a time. 1281 * <p> 1282 * However, if we have any auto-versioned references we do this in 2 passes: First the resources from the 1283 * transaction that don't have any auto-versioned references are stored. We do them first since there's 1284 * a chance they may be a NOP and we'll need to account for their version number not actually changing. 1285 * Then we do a second pass for any resources that have auto-versioned references. These happen in a separate 1286 * pass because it's too complex to try and insert the auto-versioned references and still 1287 * account for NOPs, so we block NOPs in that pass. 1288 */ 1289 private void resolveReferencesThenSaveAndIndexResources(RequestDetails theRequest, TransactionDetails theTransactionDetails, 1290 IdSubstitutionMap theIdSubstitutions, Map<IIdType, DaoMethodOutcome> theIdToPersistedOutcome, 1291 StopWatch theTransactionStopWatch, EntriesToProcessMap entriesToProcess, 1292 Set<IIdType> nonUpdatedEntities, Set<IBasePersistedResource> updatedEntities) { 1293 FhirTerser terser = myContext.newTerser(); 1294 theTransactionStopWatch.startTask("Index " + theIdToPersistedOutcome.size() + " resources"); 1295 IdentityHashMap<DaoMethodOutcome, Set<IBaseReference>> deferredIndexesForAutoVersioning = null; 1296 int i = 0; 1297 for (DaoMethodOutcome nextOutcome : theIdToPersistedOutcome.values()) { 1298 1299 if (i++ % 250 == 0) { 1300 ourLog.debug("Have indexed {} entities out of {} in transaction", i, theIdToPersistedOutcome.values().size()); 1301 } 1302 1303 if (nextOutcome.isNop()) { 1304 continue; 1305 } 1306 1307 IBaseResource nextResource = nextOutcome.getResource(); 1308 if (nextResource == null) { 1309 continue; 1310 } 1311 1312 Set<IBaseReference> referencesToAutoVersion = BaseStorageDao.extractReferencesToAutoVersion(myContext, myModelConfig, nextResource); 1313 if (referencesToAutoVersion.isEmpty()) { 1314 // no references to autoversion - we can do the resolve and save now 1315 resolveReferencesThenSaveAndIndexResource(theRequest, theTransactionDetails, 1316 theIdSubstitutions, theIdToPersistedOutcome, 1317 entriesToProcess, nonUpdatedEntities, 1318 updatedEntities, terser, 1319 nextOutcome, nextResource, 1320 referencesToAutoVersion); // this is empty 1321 } else { 1322 // we have autoversioned things to defer until later 1323 if (deferredIndexesForAutoVersioning == null) { 1324 deferredIndexesForAutoVersioning = new IdentityHashMap<>(); 1325 } 1326 deferredIndexesForAutoVersioning.put(nextOutcome, referencesToAutoVersion); 1327 } 1328 } 1329 1330 // If we have any resources we'll be auto-versioning, index these next 1331 if (deferredIndexesForAutoVersioning != null) { 1332 for (Map.Entry<DaoMethodOutcome, Set<IBaseReference>> nextEntry : deferredIndexesForAutoVersioning.entrySet()) { 1333 DaoMethodOutcome nextOutcome = nextEntry.getKey(); 1334 Set<IBaseReference> referencesToAutoVersion = nextEntry.getValue(); 1335 IBaseResource nextResource = nextOutcome.getResource(); 1336 1337 1338 resolveReferencesThenSaveAndIndexResource(theRequest, theTransactionDetails, 1339 theIdSubstitutions, theIdToPersistedOutcome, 1340 entriesToProcess, nonUpdatedEntities, 1341 updatedEntities, terser, 1342 nextOutcome, nextResource, 1343 referencesToAutoVersion); 1344 } 1345 } 1346 } 1347 1348 private void resolveReferencesThenSaveAndIndexResource(RequestDetails theRequest, TransactionDetails theTransactionDetails, 1349 IdSubstitutionMap theIdSubstitutions, Map<IIdType, DaoMethodOutcome> theIdToPersistedOutcome, 1350 EntriesToProcessMap entriesToProcess, Set<IIdType> nonUpdatedEntities, 1351 Set<IBasePersistedResource> updatedEntities, FhirTerser terser, 1352 DaoMethodOutcome nextOutcome, IBaseResource nextResource, 1353 Set<IBaseReference> theReferencesToAutoVersion) { 1354 // References 1355 List<ResourceReferenceInfo> allRefs = terser.getAllResourceReferences(nextResource); 1356 for (ResourceReferenceInfo nextRef : allRefs) { 1357 IBaseReference resourceReference = nextRef.getResourceReference(); 1358 IIdType nextId = resourceReference.getReferenceElement(); 1359 IIdType newId = null; 1360 if (!nextId.hasIdPart()) { 1361 if (resourceReference.getResource() != null) { 1362 IIdType targetId = resourceReference.getResource().getIdElement(); 1363 if (targetId.getValue() == null || targetId.getValue().startsWith("#")) { 1364 // This means it's a contained resource 1365 continue; 1366 } else if (theIdSubstitutions.containsTarget(targetId)) { 1367 newId = targetId; 1368 } else { 1369 throw new InternalErrorException(Msg.code(540) + "References by resource with no reference ID are not supported in DAO layer"); 1370 } 1371 } else { 1372 continue; 1373 } 1374 } 1375 if (newId != null || theIdSubstitutions.containsSource(nextId)) { 1376 if (newId == null) { 1377 newId = theIdSubstitutions.getForSource(nextId); 1378 } 1379 if (newId != null) { 1380 ourLog.debug(" * Replacing resource ref {} with {}", nextId, newId); 1381 1382 addRollbackReferenceRestore(theTransactionDetails, resourceReference); 1383 if (theReferencesToAutoVersion.contains(resourceReference)) { 1384 resourceReference.setReference(newId.getValue()); 1385 resourceReference.setResource(null); 1386 } else { 1387 resourceReference.setReference(newId.toVersionless().getValue()); 1388 resourceReference.setResource(null); 1389 } 1390 } 1391 } else if (nextId.getValue().startsWith("urn:")) { 1392 throw new InvalidRequestException(Msg.code(541) + "Unable to satisfy placeholder ID " + nextId.getValue() + " found in element named '" + nextRef.getName() + "' within resource of type: " + nextResource.getIdElement().getResourceType()); 1393 } else { 1394 // get a map of 1395 // existing ids -> PID (for resources that exist in the DB) 1396 // should this be allPartitions? 1397 ResourcePersistentIdMap resourceVersionMap = myResourceVersionSvc.getLatestVersionIdsForResourceIds(RequestPartitionId.allPartitions(), 1398 theReferencesToAutoVersion.stream() 1399 .map(IBaseReference::getReferenceElement).collect(Collectors.toList())); 1400 1401 for (IBaseReference baseRef : theReferencesToAutoVersion) { 1402 IIdType id = baseRef.getReferenceElement(); 1403 if (!resourceVersionMap.containsKey(id) 1404 && myDaoConfig.isAutoCreatePlaceholderReferenceTargets()) { 1405 // not in the db, but autocreateplaceholders is true 1406 // so the version we'll set is "1" (since it will be 1407 // created later) 1408 String newRef = id.withVersion("1").getValue(); 1409 id.setValue(newRef); 1410 } else { 1411 // we will add the looked up info to the transaction 1412 // for later 1413 theTransactionDetails.addResolvedResourceId(id, 1414 resourceVersionMap.getResourcePersistentId(id)); 1415 } 1416 } 1417 1418 if (theReferencesToAutoVersion.contains(resourceReference)) { 1419 DaoMethodOutcome outcome = theIdToPersistedOutcome.get(nextId); 1420 1421 if (outcome != null && !outcome.isNop() && !Boolean.TRUE.equals(outcome.getCreated())) { 1422 addRollbackReferenceRestore(theTransactionDetails, resourceReference); 1423 resourceReference.setReference(nextId.getValue()); 1424 resourceReference.setResource(null); 1425 } 1426 } 1427 } 1428 } 1429 1430 // URIs 1431 Class<? extends IPrimitiveType<?>> uriType = (Class<? extends IPrimitiveType<?>>) myContext.getElementDefinition("uri").getImplementingClass(); 1432 List<? extends IPrimitiveType<?>> allUris = terser.getAllPopulatedChildElementsOfType(nextResource, uriType); 1433 for (IPrimitiveType<?> nextRef : allUris) { 1434 if (nextRef instanceof IIdType) { 1435 continue; // No substitution on the resource ID itself! 1436 } 1437 String nextUriString = nextRef.getValueAsString(); 1438 if (theIdSubstitutions.containsSource(nextUriString)) { 1439 IIdType newId = theIdSubstitutions.getForSource(nextUriString); 1440 ourLog.debug(" * Replacing resource ref {} with {}", nextUriString, newId); 1441 1442 String existingValue = nextRef.getValueAsString(); 1443 theTransactionDetails.addRollbackUndoAction(() -> nextRef.setValueAsString(existingValue)); 1444 1445 nextRef.setValueAsString(newId.toVersionless().getValue()); 1446 } else { 1447 ourLog.debug(" * Reference [{}] does not exist in bundle", nextUriString); 1448 } 1449 } 1450 1451 IPrimitiveType<Date> deletedInstantOrNull; 1452 if (nextResource instanceof IAnyResource) { 1453 deletedInstantOrNull = ResourceMetadataKeyEnum.DELETED_AT.get((IAnyResource) nextResource); 1454 } else { 1455 deletedInstantOrNull = ResourceMetadataKeyEnum.DELETED_AT.get((IResource) nextResource); 1456 } 1457 Date deletedTimestampOrNull = deletedInstantOrNull != null ? deletedInstantOrNull.getValue() : null; 1458 1459 IFhirResourceDao<? extends IBaseResource> dao = myDaoRegistry.getResourceDao(nextResource.getClass()); 1460 IJpaDao jpaDao = (IJpaDao) dao; 1461 1462 IBasePersistedResource updateOutcome = null; 1463 if (updatedEntities.contains(nextOutcome.getEntity())) { 1464 boolean forceUpdateVersion = !theReferencesToAutoVersion.isEmpty(); 1465 1466 updateOutcome = jpaDao.updateInternal(theRequest, nextResource, true, forceUpdateVersion, nextOutcome.getEntity(), nextResource.getIdElement(), nextOutcome.getPreviousResource(), theTransactionDetails); 1467 } else if (!nonUpdatedEntities.contains(nextOutcome.getId())) { 1468 updateOutcome = jpaDao.updateEntity(theRequest, nextResource, nextOutcome.getEntity(), deletedTimestampOrNull, true, false, theTransactionDetails, false, true); 1469 } 1470 1471 // Make sure we reflect the actual final version for the resource. 1472 if (updateOutcome != null) { 1473 IIdType newId = updateOutcome.getIdDt(); 1474 1475 IIdType entryId = entriesToProcess.getIdWithVersionlessComparison(newId); 1476 if (entryId != null && !StringUtils.equals(entryId.getValue(), newId.getValue())) { 1477 entryId.setValue(newId.getValue()); 1478 } 1479 1480 nextOutcome.setId(newId); 1481 1482 IIdType target = theIdSubstitutions.getForSource(newId); 1483 if (target != null) { 1484 target.setValue(newId.getValue()); 1485 } 1486 1487 } 1488 } 1489 1490 private void addRollbackReferenceRestore(TransactionDetails theTransactionDetails, IBaseReference resourceReference) { 1491 String existingValue = resourceReference.getReferenceElement().getValue(); 1492 theTransactionDetails.addRollbackUndoAction(() -> resourceReference.setReference(existingValue)); 1493 } 1494 1495 private void validateNoDuplicates(RequestDetails theRequest, String theActionName, Map<String, Class<? extends IBaseResource>> conditionalRequestUrls, Collection<DaoMethodOutcome> thePersistedOutcomes) { 1496 1497 IdentityHashMap<IBaseResource, ResourceIndexedSearchParams> resourceToIndexedParams = new IdentityHashMap<>(thePersistedOutcomes.size()); 1498 thePersistedOutcomes 1499 .stream() 1500 .filter(t -> !t.isNop()) 1501 .filter(t -> t.getEntity() instanceof ResourceTable)//N.B. GGG: This validation never occurs for mongo, as nothing is a ResourceTable. 1502 .filter(t -> t.getEntity().getDeleted() == null) 1503 .filter(t -> t.getResource() != null) 1504 .forEach(t -> resourceToIndexedParams.put(t.getResource(), new ResourceIndexedSearchParams((ResourceTable) t.getEntity()))); 1505 1506 for (Map.Entry<String, Class<? extends IBaseResource>> nextEntry : conditionalRequestUrls.entrySet()) { 1507 String matchUrl = nextEntry.getKey(); 1508 if (isNotBlank(matchUrl)) { 1509 if (matchUrl.startsWith("?") || (!matchUrl.contains("?") && UNQUALIFIED_MATCH_URL_START.matcher(matchUrl).find())) { 1510 StringBuilder b = new StringBuilder(); 1511 b.append(myContext.getResourceType(nextEntry.getValue())); 1512 if (!matchUrl.startsWith("?")) { 1513 b.append("?"); 1514 } 1515 b.append(matchUrl); 1516 matchUrl = b.toString(); 1517 } 1518 1519 if (!myInMemoryResourceMatcher.canBeEvaluatedInMemory(matchUrl).supported()) { 1520 continue; 1521 } 1522 1523 int counter = 0; 1524 for (Map.Entry<IBaseResource, ResourceIndexedSearchParams> entries : resourceToIndexedParams.entrySet()) { 1525 ResourceIndexedSearchParams indexedParams = entries.getValue(); 1526 IBaseResource resource = entries.getKey(); 1527 1528 String resourceType = myContext.getResourceType(resource); 1529 if (!matchUrl.startsWith(resourceType + "?")) { 1530 continue; 1531 } 1532 1533 if (myInMemoryResourceMatcher.match(matchUrl, resource, indexedParams).matched()) { 1534 counter++; 1535 if (counter > 1) { 1536 throw new InvalidRequestException(Msg.code(542) + "Unable to process " + theActionName + " - Request would cause multiple resources to match URL: \"" + matchUrl + "\". Does transaction request contain duplicates?"); 1537 } 1538 } 1539 } 1540 } 1541 } 1542 } 1543 1544 protected abstract void flushSession(Map<IIdType, DaoMethodOutcome> theIdToPersistedOutcome); 1545 1546 private void validateResourcePresent(IBaseResource theResource, Integer theOrder, String theVerb) { 1547 if (theResource == null) { 1548 String msg = myContext.getLocalizer().getMessage(BaseTransactionProcessor.class, "missingMandatoryResource", theVerb, theOrder); 1549 throw new InvalidRequestException(Msg.code(543) + msg); 1550 } 1551 } 1552 1553 private IIdType newIdType(String theResourceType, String theResourceId, String theVersion) { 1554 org.hl7.fhir.r4.model.IdType id = new org.hl7.fhir.r4.model.IdType(theResourceType, theResourceId, theVersion); 1555 return myContext.getVersion().newIdType().setValue(id.getValue()); 1556 } 1557 1558 private IIdType newIdType(String theToResourceName, String theIdPart) { 1559 return newIdType(theToResourceName, theIdPart, null); 1560 } 1561 1562 @VisibleForTesting 1563 public void setDaoRegistry(DaoRegistry theDaoRegistry) { 1564 myDaoRegistry = theDaoRegistry; 1565 } 1566 1567 private IFhirResourceDao getDaoOrThrowException(Class<? extends IBaseResource> theClass) { 1568 IFhirResourceDao<? extends IBaseResource> dao = myDaoRegistry.getResourceDaoOrNull(theClass); 1569 if (dao == null) { 1570 Set<String> types = new TreeSet<>(myDaoRegistry.getRegisteredDaoTypes()); 1571 String type = myContext.getResourceType(theClass); 1572 String msg = myContext.getLocalizer().getMessage(BaseTransactionProcessor.class, "unsupportedResourceType", type, types.toString()); 1573 throw new InvalidRequestException(Msg.code(544) + msg); 1574 } 1575 return dao; 1576 } 1577 1578 private String toResourceName(Class<? extends IBaseResource> theResourceType) { 1579 return myContext.getResourceType(theResourceType); 1580 } 1581 1582 public void setContext(FhirContext theContext) { 1583 myContext = theContext; 1584 } 1585 1586 /** 1587 * Extracts the transaction url from the entry and verifies it's: 1588 * * not null or bloack 1589 * * is a relative url matching the resourceType it is about 1590 * 1591 * Returns the transaction url (or throws an InvalidRequestException if url is not valid) 1592 */ 1593 private String extractAndVerifyTransactionUrlForEntry(IBase theEntry, String theVerb) { 1594 String url = extractTransactionUrlOrThrowException(theEntry, theVerb); 1595 1596 if (!isValidResourceTypeUrl(url)) { 1597 ourLog.debug("Invalid url. Should begin with a resource type: {}", url); 1598 String msg = myContext.getLocalizer().getMessage(BaseStorageDao.class, "transactionInvalidUrl", theVerb, url); 1599 throw new InvalidRequestException(Msg.code(2006) + msg); 1600 } 1601 return url; 1602 } 1603 1604 /** 1605 * Returns true if the provided url is a valid entry request.url. 1606 * 1607 * This means: 1608 * a) not an absolute url (does not start with http/https) 1609 * b) starts with either a ResourceType or /ResourceType 1610 */ 1611 private boolean isValidResourceTypeUrl(@Nonnull String theUrl) { 1612 if (UrlUtil.isAbsolute(theUrl)) { 1613 return false; 1614 } else { 1615 int queryStringIndex = theUrl.indexOf("?"); 1616 String url; 1617 if (queryStringIndex > 0) { 1618 url = theUrl.substring(0, theUrl.indexOf("?")); 1619 } else { 1620 url = theUrl; 1621 } 1622 String[] parts; 1623 if (url.startsWith("/")) { 1624 parts = url.substring(1).split("/"); 1625 } else { 1626 parts = url.split("/"); 1627 } 1628 Set<String> allResourceTypes = myContext.getResourceTypes(); 1629 1630 return allResourceTypes.contains(parts[0]); 1631 } 1632 } 1633 1634 /** 1635 * Extracts the transaction url from the entry and verifies that it is not null/blank 1636 * and returns it 1637 */ 1638 private String extractTransactionUrlOrThrowException(IBase nextEntry, String verb) { 1639 String url = myVersionAdapter.getEntryRequestUrl(nextEntry); 1640 if (isBlank(url)) { 1641 throw new InvalidRequestException(Msg.code(545) + myContext.getLocalizer().getMessage(BaseStorageDao.class, "transactionMissingUrl", verb)); 1642 } 1643 return url; 1644 } 1645 1646 private IFhirResourceDao<? extends IBaseResource> toDao(UrlUtil.UrlParts theParts, String theVerb, String theUrl) { 1647 RuntimeResourceDefinition resType; 1648 try { 1649 resType = myContext.getResourceDefinition(theParts.getResourceType()); 1650 } catch (DataFormatException e) { 1651 String msg = myContext.getLocalizer().getMessage(BaseStorageDao.class, "transactionInvalidUrl", theVerb, theUrl); 1652 throw new InvalidRequestException(Msg.code(546) + msg); 1653 } 1654 IFhirResourceDao<? extends IBaseResource> dao = null; 1655 if (resType != null) { 1656 dao = myDaoRegistry.getResourceDao(resType.getImplementingClass()); 1657 } 1658 if (dao == null) { 1659 String msg = myContext.getLocalizer().getMessage(BaseStorageDao.class, "transactionInvalidUrl", theVerb, theUrl); 1660 throw new InvalidRequestException(Msg.code(547) + msg); 1661 } 1662 1663 return dao; 1664 } 1665 1666 private String toMatchUrl(IBase theEntry) { 1667 String verb = myVersionAdapter.getEntryRequestVerb(myContext, theEntry); 1668 if (verb.equals("POST")) { 1669 return myVersionAdapter.getEntryIfNoneExist(theEntry); 1670 } 1671 if (verb.equals("PATCH")) { 1672 return myVersionAdapter.getEntryRequestIfMatch(theEntry); 1673 } 1674 if (verb.equals("PUT") || verb.equals("DELETE")) { 1675 String url = extractTransactionUrlOrThrowException(theEntry, verb); 1676 UrlUtil.UrlParts parts = UrlUtil.parseUrl(url); 1677 if (isBlank(parts.getResourceId())) { 1678 return parts.getResourceType() + '?' + parts.getParams(); 1679 } 1680 } 1681 return null; 1682 } 1683 1684 /** 1685 * Transaction Order, per the spec: 1686 * <p> 1687 * Process any DELETE interactions 1688 * Process any POST interactions 1689 * Process any PUT interactions 1690 * Process any PATCH interactions 1691 * Process any GET interactions 1692 */ 1693 //@formatter:off 1694 public class TransactionSorter implements Comparator<IBase> { 1695 1696 private final Set<String> myPlaceholderIds; 1697 1698 public TransactionSorter(Set<String> thePlaceholderIds) { 1699 myPlaceholderIds = thePlaceholderIds; 1700 } 1701 1702 @Override 1703 public int compare(IBase theO1, IBase theO2) { 1704 int o1 = toOrder(theO1); 1705 int o2 = toOrder(theO2); 1706 1707 if (o1 == o2) { 1708 String matchUrl1 = toMatchUrl(theO1); 1709 String matchUrl2 = toMatchUrl(theO2); 1710 if (isBlank(matchUrl1) && isBlank(matchUrl2)) { 1711 return 0; 1712 } 1713 if (isBlank(matchUrl1)) { 1714 return -1; 1715 } 1716 if (isBlank(matchUrl2)) { 1717 return 1; 1718 } 1719 1720 boolean match1containsSubstitutions = false; 1721 boolean match2containsSubstitutions = false; 1722 for (String nextPlaceholder : myPlaceholderIds) { 1723 if (matchUrl1.contains(nextPlaceholder)) { 1724 match1containsSubstitutions = true; 1725 } 1726 if (matchUrl2.contains(nextPlaceholder)) { 1727 match2containsSubstitutions = true; 1728 } 1729 } 1730 1731 if (match1containsSubstitutions && match2containsSubstitutions) { 1732 return 0; 1733 } 1734 if (!match1containsSubstitutions && !match2containsSubstitutions) { 1735 return 0; 1736 } 1737 if (match1containsSubstitutions) { 1738 return 1; 1739 } else { 1740 return -1; 1741 } 1742 } 1743 1744 return o1 - o2; 1745 } 1746 1747 private int toOrder(IBase theO1) { 1748 int o1 = 0; 1749 if (myVersionAdapter.getEntryRequestVerb(myContext, theO1) != null) { 1750 switch (myVersionAdapter.getEntryRequestVerb(myContext, theO1)) { 1751 case "DELETE": 1752 o1 = 1; 1753 break; 1754 case "POST": 1755 o1 = 2; 1756 break; 1757 case "PUT": 1758 o1 = 3; 1759 break; 1760 case "PATCH": 1761 o1 = 4; 1762 break; 1763 case "GET": 1764 o1 = 5; 1765 break; 1766 default: 1767 o1 = 0; 1768 break; 1769 } 1770 } 1771 return o1; 1772 } 1773 1774 } 1775 1776 public class RetriableBundleTask implements Runnable { 1777 1778 private final CountDownLatch myCompletedLatch; 1779 private final RequestDetails myRequestDetails; 1780 private final IBase myNextReqEntry; 1781 private final Map<Integer, Object> myResponseMap; 1782 private final int myResponseOrder; 1783 private final boolean myNestedMode; 1784 private BaseServerResponseException myLastSeenException; 1785 1786 protected RetriableBundleTask(CountDownLatch theCompletedLatch, RequestDetails theRequestDetails, Map<Integer, Object> theResponseMap, int theResponseOrder, IBase theNextReqEntry, boolean theNestedMode) { 1787 this.myCompletedLatch = theCompletedLatch; 1788 this.myRequestDetails = theRequestDetails; 1789 this.myNextReqEntry = theNextReqEntry; 1790 this.myResponseMap = theResponseMap; 1791 this.myResponseOrder = theResponseOrder; 1792 this.myNestedMode = theNestedMode; 1793 this.myLastSeenException = null; 1794 } 1795 1796 private void processBatchEntry() { 1797 IBaseBundle subRequestBundle = myVersionAdapter.createBundle(org.hl7.fhir.r4.model.Bundle.BundleType.TRANSACTION.toCode()); 1798 myVersionAdapter.addEntry(subRequestBundle, myNextReqEntry); 1799 1800 IBaseBundle nextResponseBundle = processTransactionAsSubRequest(myRequestDetails, subRequestBundle, "Batch sub-request", myNestedMode); 1801 1802 IBase subResponseEntry = (IBase) myVersionAdapter.getEntries(nextResponseBundle).get(0); 1803 myResponseMap.put(myResponseOrder, subResponseEntry); 1804 1805 /* 1806 * If the individual entry didn't have a resource in its response, bring the sub-transaction's OperationOutcome across so the client can see it 1807 */ 1808 if (myVersionAdapter.getResource(subResponseEntry) == null) { 1809 IBase nextResponseBundleFirstEntry = (IBase) myVersionAdapter.getEntries(nextResponseBundle).get(0); 1810 myResponseMap.put(myResponseOrder, nextResponseBundleFirstEntry); 1811 } 1812 } 1813 1814 private boolean processBatchEntryWithRetry() { 1815 int maxAttempts = 3; 1816 for (int attempt = 1; ; attempt++) { 1817 try { 1818 processBatchEntry(); 1819 return true; 1820 } catch (BaseServerResponseException e) { 1821 //If we catch a known and structured exception from HAPI, just fail. 1822 myLastSeenException = e; 1823 return false; 1824 } catch (Throwable t) { 1825 myLastSeenException = new InternalErrorException(t); 1826 //If we have caught a non-tag-storage failure we are unfamiliar with, or we have exceeded max attempts, exit. 1827 if (!DaoFailureUtil.isTagStorageFailure(t) || attempt >= maxAttempts) { 1828 ourLog.error("Failure during BATCH sub transaction processing", t); 1829 return false; 1830 } 1831 } 1832 } 1833 } 1834 1835 @Override 1836 public void run() { 1837 boolean success = processBatchEntryWithRetry(); 1838 if (!success) { 1839 populateResponseMapWithLastSeenException(); 1840 } 1841 1842 // checking for the parallelism 1843 ourLog.debug("processing batch for {} is completed", myVersionAdapter.getEntryRequestUrl(myNextReqEntry)); 1844 myCompletedLatch.countDown(); 1845 } 1846 1847 private void populateResponseMapWithLastSeenException() { 1848 BaseServerResponseExceptionHolder caughtEx = new BaseServerResponseExceptionHolder(); 1849 caughtEx.setException(myLastSeenException); 1850 myResponseMap.put(myResponseOrder, caughtEx); 1851 } 1852 1853 } 1854 1855 private static class BaseServerResponseExceptionHolder { 1856 private BaseServerResponseException myException; 1857 1858 public BaseServerResponseException getException() { 1859 return myException; 1860 } 1861 1862 public void setException(BaseServerResponseException myException) { 1863 this.myException = myException; 1864 } 1865 } 1866 1867 public static boolean isPlaceholder(IIdType theId) { 1868 if (theId != null && theId.getValue() != null) { 1869 return theId.getValue().startsWith("urn:oid:") || theId.getValue().startsWith("urn:uuid:"); 1870 } 1871 return false; 1872 } 1873 1874 private static String toStatusString(int theStatusCode) { 1875 return theStatusCode + " " + defaultString(Constants.HTTP_STATUS_NAMES.get(theStatusCode)); 1876 } 1877 1878 /** 1879 * Given a match URL containing 1880 * 1881 * @param theIdSubstitutions 1882 * @param theMatchUrl 1883 * @return 1884 */ 1885 static String performIdSubstitutionsInMatchUrl(IdSubstitutionMap theIdSubstitutions, String theMatchUrl) { 1886 String matchUrl = theMatchUrl; 1887 if (isNotBlank(matchUrl) && !theIdSubstitutions.isEmpty()) { 1888 1889 int startIdx = matchUrl.indexOf('?'); 1890 while (startIdx != -1) { 1891 1892 int endIdx = matchUrl.indexOf('&', startIdx + 1); 1893 if (endIdx == -1) { 1894 endIdx = matchUrl.length(); 1895 } 1896 1897 int equalsIdx = matchUrl.indexOf('=', startIdx + 1); 1898 1899 int searchFrom; 1900 if (equalsIdx == -1) { 1901 searchFrom = matchUrl.length(); 1902 } else if (equalsIdx >= endIdx) { 1903 // First equals we found is from a subsequent parameter 1904 searchFrom = matchUrl.length(); 1905 } else { 1906 String paramValue = matchUrl.substring(equalsIdx + 1, endIdx); 1907 boolean isUrn = isUrn(paramValue); 1908 boolean isUrnEscaped = !isUrn && isUrnEscaped(paramValue); 1909 if (isUrn || isUrnEscaped) { 1910 if (isUrnEscaped) { 1911 paramValue = UrlUtil.unescape(paramValue); 1912 } 1913 IIdType replacement = theIdSubstitutions.getForSource(paramValue); 1914 if (replacement != null) { 1915 String replacementValue; 1916 if (replacement.hasVersionIdPart()) { 1917 replacementValue = replacement.toVersionless().getValue(); 1918 } else { 1919 replacementValue = replacement.getValue(); 1920 } 1921 matchUrl = matchUrl.substring(0, equalsIdx + 1) + replacementValue + matchUrl.substring(endIdx); 1922 searchFrom = equalsIdx + 1 + replacementValue.length(); 1923 } else { 1924 searchFrom = endIdx; 1925 } 1926 } else { 1927 searchFrom = endIdx; 1928 } 1929 } 1930 1931 if (searchFrom >= matchUrl.length()) { 1932 break; 1933 } 1934 1935 startIdx = matchUrl.indexOf('&', searchFrom); 1936 } 1937 1938 } 1939 return matchUrl; 1940 } 1941 1942 private static boolean isUrn(@Nonnull String theId) { 1943 return theId.startsWith(URN_PREFIX); 1944 } 1945 1946 private static boolean isUrnEscaped(@Nonnull String theId) { 1947 return theId.startsWith(URN_PREFIX_ESCAPED); 1948 } 1949}