Source code

001package ca.uhn.fhir.jpa.bulk.export.provider;
002
003/*-
004 * #%L
005 * HAPI FHIR Storage api
006 * %%
007 * Copyright (C) 2014 - 2022 Smile CDR, Inc.
008 * %%
009 * Licensed under the Apache License, Version 2.0 (the "License");
010 * you may not use this file except in compliance with the License.
011 * You may obtain a copy of the License at
012 *
013 *      http://www.apache.org/licenses/LICENSE-2.0
014 *
015 * Unless required by applicable law or agreed to in writing, software
016 * distributed under the License is distributed on an "AS IS" BASIS,
017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
018 * See the License for the specific language governing permissions and
019 * limitations under the License.
020 * #L%
021 */
022
023import ca.uhn.fhir.i18n.Msg;
024import ca.uhn.fhir.context.FhirContext;
025import ca.uhn.fhir.jpa.bulk.export.api.IBulkDataExportSvc;
026import ca.uhn.fhir.jpa.bulk.export.model.BulkExportResponseJson;
027import ca.uhn.fhir.jpa.model.util.JpaConstants;
028import ca.uhn.fhir.rest.annotation.IdParam;
029import ca.uhn.fhir.rest.annotation.Operation;
030import ca.uhn.fhir.rest.annotation.OperationParam;
031import ca.uhn.fhir.rest.api.CacheControlDirective;
032import ca.uhn.fhir.rest.api.Constants;
033import ca.uhn.fhir.rest.api.PreferHeader;
034import ca.uhn.fhir.rest.api.server.bulk.BulkDataExportOptions;
035import ca.uhn.fhir.rest.server.RestfulServerUtils;
036import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
037import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
038import ca.uhn.fhir.util.ArrayUtil;
039import ca.uhn.fhir.util.JsonUtil;
040import ca.uhn.fhir.util.OperationOutcomeUtil;
041import com.google.common.annotations.VisibleForTesting;
042import org.apache.commons.lang3.StringUtils;
043import org.hl7.fhir.instance.model.api.IBaseOperationOutcome;
044import org.hl7.fhir.instance.model.api.IIdType;
045import org.hl7.fhir.instance.model.api.IPrimitiveType;
046import org.hl7.fhir.r4.model.InstantType;
047import org.slf4j.Logger;
048import org.springframework.beans.factory.annotation.Autowired;
049
050import javax.servlet.http.HttpServletResponse;
051import java.io.IOException;
052import java.util.Arrays;
053import java.util.Date;
054import java.util.HashSet;
055import java.util.List;
056import java.util.Set;
057import java.util.stream.Collectors;
058
059import static org.slf4j.LoggerFactory.getLogger;
060
061
062public class BulkDataExportProvider {
063        public static final String FARM_TO_TABLE_TYPE_FILTER_REGEX = "(?:,)(?=[A-Z][a-z]+\\?)";
064        private static final Logger ourLog = getLogger(BulkDataExportProvider.class);
065
066        @Autowired
067        private IBulkDataExportSvc myBulkDataExportSvc;
068        @Autowired
069        private FhirContext myFhirContext;
070
071        @VisibleForTesting
072        public void setFhirContextForUnitTest(FhirContext theFhirContext) {
073                myFhirContext = theFhirContext;
074        }
075
076        @VisibleForTesting
077        public void setBulkDataExportSvcForUnitTests(IBulkDataExportSvc theBulkDataExportSvc) {
078                myBulkDataExportSvc = theBulkDataExportSvc;
079        }
080
081        /**
082         * $export
083         */
084        @Operation(name = JpaConstants.OPERATION_EXPORT, global = false /* set to true once we can handle this */, manualResponse = true, idempotent = true)
085        public void export(
086                @OperationParam(name = JpaConstants.PARAM_EXPORT_OUTPUT_FORMAT, min = 0, max = 1, typeName = "string") IPrimitiveType<String> theOutputFormat,
087                @OperationParam(name = JpaConstants.PARAM_EXPORT_TYPE, min = 0, max = 1, typeName = "string") IPrimitiveType<String> theType,
088                @OperationParam(name = JpaConstants.PARAM_EXPORT_SINCE, min = 0, max = 1, typeName = "instant") IPrimitiveType<Date> theSince,
089                @OperationParam(name = JpaConstants.PARAM_EXPORT_TYPE_FILTER, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "string") List<IPrimitiveType<String>> theTypeFilter,
090                ServletRequestDetails theRequestDetails
091        ) {
092                validatePreferAsyncHeader(theRequestDetails);
093                BulkDataExportOptions bulkDataExportOptions = buildSystemBulkExportOptions(theOutputFormat, theType, theSince, theTypeFilter);
094                Boolean useCache = shouldUseCache(theRequestDetails);
095                IBulkDataExportSvc.JobInfo outcome = myBulkDataExportSvc.submitJob(bulkDataExportOptions, useCache, theRequestDetails);
096                writePollingLocationToResponseHeaders(theRequestDetails, outcome);
097        }
098
099        private boolean shouldUseCache(ServletRequestDetails theRequestDetails) {
100                CacheControlDirective cacheControlDirective = new CacheControlDirective().parse(theRequestDetails.getHeaders(Constants.HEADER_CACHE_CONTROL));
101                return !cacheControlDirective.isNoCache();
102        }
103
104        private String getServerBase(ServletRequestDetails theRequestDetails) {
105                return StringUtils.removeEnd(theRequestDetails.getServerBaseForRequest(), "/");
106        }
107
108        /**
109         * Group/Id/$export
110         */
111        @Operation(name = JpaConstants.OPERATION_EXPORT, manualResponse = true, idempotent = true, typeName = "Group")
112        public void groupExport(
113                @IdParam IIdType theIdParam,
114                @OperationParam(name = JpaConstants.PARAM_EXPORT_OUTPUT_FORMAT, min = 0, max = 1, typeName = "string") IPrimitiveType<String> theOutputFormat,
115                @OperationParam(name = JpaConstants.PARAM_EXPORT_TYPE, min = 0, max = 1, typeName = "string") IPrimitiveType<String> theType,
116                @OperationParam(name = JpaConstants.PARAM_EXPORT_SINCE, min = 0, max = 1, typeName = "instant") IPrimitiveType<Date> theSince,
117                @OperationParam(name = JpaConstants.PARAM_EXPORT_TYPE_FILTER, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "string") List<IPrimitiveType<String>> theTypeFilter,
118                @OperationParam(name = JpaConstants.PARAM_EXPORT_MDM, min = 0, max = 1, typeName = "boolean") IPrimitiveType<Boolean> theMdm,
119                ServletRequestDetails theRequestDetails
120        ) {
121                ourLog.debug("Received Group Bulk Export Request for Group {}", theIdParam);
122                ourLog.debug("_type={}", theIdParam);
123                ourLog.debug("_since={}", theSince);
124                ourLog.debug("_typeFilter={}", theTypeFilter);
125                ourLog.debug("_mdm=", theMdm);
126
127
128                validatePreferAsyncHeader(theRequestDetails);
129                BulkDataExportOptions bulkDataExportOptions = buildGroupBulkExportOptions(theOutputFormat, theType, theSince, theTypeFilter, theIdParam, theMdm);
130                validateResourceTypesAllContainPatientSearchParams(bulkDataExportOptions.getResourceTypes());
131                IBulkDataExportSvc.JobInfo outcome = myBulkDataExportSvc.submitJob(bulkDataExportOptions, shouldUseCache(theRequestDetails), null);
132                writePollingLocationToResponseHeaders(theRequestDetails, outcome);
133        }
134
135        private void validateResourceTypesAllContainPatientSearchParams(Set<String> theResourceTypes) {
136                if (theResourceTypes != null) {
137                        List<String> badResourceTypes = theResourceTypes.stream()
138                                .filter(resourceType -> !myBulkDataExportSvc.getPatientCompartmentResources().contains(resourceType))
139                                .collect(Collectors.toList());
140
141                        if (!badResourceTypes.isEmpty()) {
142                                throw new InvalidRequestException(Msg.code(512) + String.format("Resource types [%s] are invalid for this type of export, as they do not contain search parameters that refer to patients.", String.join(",", badResourceTypes)));
143                        }
144                }
145        }
146
147        /**
148         * Patient/$export
149         */
150        @Operation(name = JpaConstants.OPERATION_EXPORT, manualResponse = true, idempotent = true, typeName = "Patient")
151        public void patientExport(
152                @OperationParam(name = JpaConstants.PARAM_EXPORT_OUTPUT_FORMAT, min = 0, max = 1, typeName = "string") IPrimitiveType<String> theOutputFormat,
153                @OperationParam(name = JpaConstants.PARAM_EXPORT_TYPE, min = 0, max = 1, typeName = "string") IPrimitiveType<String> theType,
154                @OperationParam(name = JpaConstants.PARAM_EXPORT_SINCE, min = 0, max = 1, typeName = "instant") IPrimitiveType<Date> theSince,
155                @OperationParam(name = JpaConstants.PARAM_EXPORT_TYPE_FILTER, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "string") List<IPrimitiveType<String>> theTypeFilter,
156                ServletRequestDetails theRequestDetails
157        ) {
158                validatePreferAsyncHeader(theRequestDetails);
159                BulkDataExportOptions bulkDataExportOptions = buildPatientBulkExportOptions(theOutputFormat, theType, theSince, theTypeFilter);
160                validateResourceTypesAllContainPatientSearchParams(bulkDataExportOptions.getResourceTypes());
161                IBulkDataExportSvc.JobInfo outcome = myBulkDataExportSvc.submitJob(bulkDataExportOptions, shouldUseCache(theRequestDetails), null);
162                writePollingLocationToResponseHeaders(theRequestDetails, outcome);
163        }
164
165        /**
166         * $export-poll-status
167         */
168        @Operation(name = JpaConstants.OPERATION_EXPORT_POLL_STATUS, manualResponse = true, idempotent = true)
169        public void exportPollStatus(
170                @OperationParam(name = JpaConstants.PARAM_EXPORT_POLL_STATUS_JOB_ID, typeName = "string", min = 0, max = 1) IPrimitiveType<String> theJobId,
171                ServletRequestDetails theRequestDetails
172        ) throws IOException {
173
174                HttpServletResponse response = theRequestDetails.getServletResponse();
175                theRequestDetails.getServer().addHeadersToResponse(response);
176
177                IBulkDataExportSvc.JobInfo status = myBulkDataExportSvc.getJobInfoOrThrowResourceNotFound(theJobId.getValueAsString());
178
179                switch (status.getStatus()) {
180                        case SUBMITTED:
181                        case BUILDING:
182
183                                response.setStatus(Constants.STATUS_HTTP_202_ACCEPTED);
184                                response.addHeader(Constants.HEADER_X_PROGRESS, "Build in progress - Status set to " + status.getStatus() + " at " + new InstantType(status.getStatusTime()).getValueAsString());
185                                response.addHeader(Constants.HEADER_RETRY_AFTER, "120");
186                                break;
187
188                        case COMPLETE:
189
190                                response.setStatus(Constants.STATUS_HTTP_200_OK);
191                                response.setContentType(Constants.CT_JSON);
192
193                                // Create a JSON response
194                                BulkExportResponseJson bulkResponseDocument = new BulkExportResponseJson();
195                                bulkResponseDocument.setTransactionTime(status.getStatusTime());
196                                bulkResponseDocument.setRequest(status.getRequest());
197                                for (IBulkDataExportSvc.FileEntry nextFile : status.getFiles()) {
198                                        String serverBase = getServerBase(theRequestDetails);
199                                        String nextUrl = serverBase + "/" + nextFile.getResourceId().toUnqualifiedVersionless().getValue();
200                                        bulkResponseDocument
201                                                .addOutput()
202                                                .setType(nextFile.getResourceType())
203                                                .setUrl(nextUrl);
204                                }
205                                JsonUtil.serialize(bulkResponseDocument, response.getWriter());
206                                response.getWriter().close();
207                                break;
208
209                        case ERROR:
210
211                                response.setStatus(Constants.STATUS_HTTP_500_INTERNAL_ERROR);
212                                response.setContentType(Constants.CT_FHIR_JSON);
213
214                                // Create an OperationOutcome response
215                                IBaseOperationOutcome oo = OperationOutcomeUtil.newInstance(myFhirContext);
216                                OperationOutcomeUtil.addIssue(myFhirContext, oo, "error", status.getStatusMessage(), null, null);
217                                myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToWriter(oo, response.getWriter());
218                                response.getWriter().close();
219                }
220        }
221
222        private BulkDataExportOptions buildSystemBulkExportOptions(IPrimitiveType<String> theOutputFormat, IPrimitiveType<String> theType, IPrimitiveType<Date> theSince, List<IPrimitiveType<String>> theTypeFilter) {
223                return buildBulkDataExportOptions(theOutputFormat, theType, theSince, theTypeFilter, BulkDataExportOptions.ExportStyle.SYSTEM);
224        }
225
226        private BulkDataExportOptions buildGroupBulkExportOptions(IPrimitiveType<String> theOutputFormat, IPrimitiveType<String> theType, IPrimitiveType<Date> theSince, List<IPrimitiveType<String>> theTypeFilter, IIdType theGroupId, IPrimitiveType<Boolean> theExpandMdm) {
227                BulkDataExportOptions bulkDataExportOptions = buildBulkDataExportOptions(theOutputFormat, theType, theSince, theTypeFilter, BulkDataExportOptions.ExportStyle.GROUP);
228                bulkDataExportOptions.setGroupId(theGroupId);
229
230                boolean mdm = false;
231                if (theExpandMdm != null) {
232                        mdm = theExpandMdm.getValue();
233                }
234                bulkDataExportOptions.setExpandMdm(mdm);
235
236                return bulkDataExportOptions;
237        }
238
239        private BulkDataExportOptions buildPatientBulkExportOptions(IPrimitiveType<String> theOutputFormat, IPrimitiveType<String> theType, IPrimitiveType<Date> theSince, List<IPrimitiveType<String>> theTypeFilter) {
240                return buildBulkDataExportOptions(theOutputFormat, theType, theSince, theTypeFilter, BulkDataExportOptions.ExportStyle.PATIENT);
241        }
242
243        private BulkDataExportOptions buildBulkDataExportOptions(IPrimitiveType<String> theOutputFormat, IPrimitiveType<String> theType, IPrimitiveType<Date> theSince, List<IPrimitiveType<String>> theTypeFilter, BulkDataExportOptions.ExportStyle theExportStyle) {
244                String outputFormat = theOutputFormat != null ? theOutputFormat.getValueAsString() : null;
245
246                Set<String> resourceTypes = null;
247                if (theType != null) {
248                        resourceTypes = ArrayUtil.commaSeparatedListToCleanSet(theType.getValueAsString());
249                }
250
251                Date since = null;
252                if (theSince != null) {
253                        since = theSince.getValue();
254                }
255
256                Set<String> typeFilters = splitTypeFilters(theTypeFilter);
257
258                BulkDataExportOptions bulkDataExportOptions = new BulkDataExportOptions();
259                bulkDataExportOptions.setFilters(typeFilters);
260                bulkDataExportOptions.setExportStyle(theExportStyle);
261                bulkDataExportOptions.setSince(since);
262                bulkDataExportOptions.setResourceTypes(resourceTypes);
263                bulkDataExportOptions.setOutputFormat(outputFormat);
264                return bulkDataExportOptions;
265        }
266
267        public void writePollingLocationToResponseHeaders(ServletRequestDetails theRequestDetails, IBulkDataExportSvc.JobInfo theOutcome) {
268                String serverBase = getServerBase(theRequestDetails);
269                String pollLocation = serverBase + "/" + JpaConstants.OPERATION_EXPORT_POLL_STATUS + "?" + JpaConstants.PARAM_EXPORT_POLL_STATUS_JOB_ID + "=" + theOutcome.getJobId();
270
271                HttpServletResponse response = theRequestDetails.getServletResponse();
272
273                // Add standard headers
274                theRequestDetails.getServer().addHeadersToResponse(response);
275
276                // Successful 202 Accepted
277                response.addHeader(Constants.HEADER_CONTENT_LOCATION, pollLocation);
278                response.setStatus(Constants.STATUS_HTTP_202_ACCEPTED);
279        }
280
281        private void validatePreferAsyncHeader(ServletRequestDetails theRequestDetails) {
282                String preferHeader = theRequestDetails.getHeader(Constants.HEADER_PREFER);
283                PreferHeader prefer = RestfulServerUtils.parsePreferHeader(null, preferHeader);
284                if (prefer.getRespondAsync() == false) {
285                        throw new InvalidRequestException(Msg.code(513) + "Must request async processing for $export");
286                }
287        }
288
289        private Set<String> splitTypeFilters(List<IPrimitiveType<String>> theTypeFilter) {
290                if (theTypeFilter== null) {
291                        return null;
292                }
293
294                Set<String> retVal = new HashSet<>();
295
296                for (IPrimitiveType<String> next : theTypeFilter) {
297                        String typeFilterString = next.getValueAsString();
298                        Arrays
299                                .stream(typeFilterString.split(FARM_TO_TABLE_TYPE_FILTER_REGEX))
300                                .filter(StringUtils::isNotBlank)
301                                .forEach(t->retVal.add(t));
302                }
303
304                return retVal;
305        }
306
307}