Source code

001package org.hl7.fhir.validation;
002
003import com.google.gson.JsonObject;
004import lombok.Getter;
005import org.hl7.fhir.convertors.conv10_50.VersionConvertor_10_50;
006import org.hl7.fhir.convertors.conv14_50.VersionConvertor_14_50;
007import org.hl7.fhir.convertors.conv30_50.VersionConvertor_30_50;
008import org.hl7.fhir.convertors.factory.VersionConvertorFactory_10_50;
009import org.hl7.fhir.convertors.factory.VersionConvertorFactory_14_50;
010import org.hl7.fhir.convertors.factory.VersionConvertorFactory_30_50;
011import org.hl7.fhir.convertors.factory.VersionConvertorFactory_40_50;
012import org.hl7.fhir.exceptions.FHIRException;
013import org.hl7.fhir.r5.context.SimpleWorkerContext;
014import org.hl7.fhir.r5.elementmodel.Manager;
015import org.hl7.fhir.r5.formats.JsonParser;
016import org.hl7.fhir.r5.formats.XmlParser;
017import org.hl7.fhir.r5.model.Constants;
018import org.hl7.fhir.r5.model.ImplementationGuide;
019import org.hl7.fhir.r5.model.Resource;
020import org.hl7.fhir.r5.utils.structuremap.StructureMapUtilities;
021import org.hl7.fhir.utilities.SimpleHTTPClient;
022import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult;
023import org.hl7.fhir.utilities.IniFile;
024import org.hl7.fhir.utilities.TextFile;
025import org.hl7.fhir.utilities.Utilities;
026import org.hl7.fhir.utilities.VersionUtilities;
027import org.hl7.fhir.utilities.json.JSONUtil;
028import org.hl7.fhir.utilities.json.JsonTrackingParser;
029import org.hl7.fhir.utilities.npm.FilesystemPackageCacheManager;
030import org.hl7.fhir.utilities.npm.NpmPackage;
031import org.hl7.fhir.utilities.turtle.Turtle;
032import org.hl7.fhir.utilities.xml.XMLUtil;
033import org.hl7.fhir.validation.cli.utils.Common;
034import org.hl7.fhir.validation.cli.utils.VersionSourceInformation;
035import org.w3c.dom.Document;
036
037import java.io.*;
038import java.net.URL;
039import java.util.ArrayList;
040import java.util.HashMap;
041import java.util.List;
042import java.util.Map;
043import java.util.zip.ZipEntry;
044import java.util.zip.ZipInputStream;
045
046public class IgLoader {
047
048  private static final String[] IGNORED_EXTENSIONS = {"md", "css", "js", "png", "gif", "jpg", "html", "tgz", "pack", "zip"};
049  private static final String[] EXEMPT_FILES = {"spec.internals", "version.info", "schematron.zip", "package.json"};
050
051  @Getter private final FilesystemPackageCacheManager packageCacheManager;
052  @Getter private final SimpleWorkerContext context;
053  @Getter private final String version;
054  @Getter private final boolean isDebug;
055
056  public IgLoader(FilesystemPackageCacheManager packageCacheManager,
057                  SimpleWorkerContext context,
058                  String theVersion) {
059      this(packageCacheManager, context, theVersion, false);
060  }
061
062  public IgLoader(FilesystemPackageCacheManager packageCacheManager,
063                  SimpleWorkerContext context,
064                  String theVersion,
065                  boolean isDebug) {
066      this.packageCacheManager = packageCacheManager;
067      this.context = context;
068      this.version = theVersion;
069      this.isDebug = isDebug;
070  }
071
072  public void loadIg(List<ImplementationGuide> igs,
073                     Map<String, byte[]> binaries,
074                     String src,
075                     boolean recursive) throws IOException, FHIRException {
076    NpmPackage npm = src.matches(FilesystemPackageCacheManager.PACKAGE_VERSION_REGEX_OPT) && !new File(src).exists() ? getPackageCacheManager().loadPackage(src, null) : null;
077    if (npm != null) {
078      for (String s : npm.dependencies()) {
079        if (!getContext().getLoadedPackages().contains(s)) {
080          if (!VersionUtilities.isCorePackage(s)) {
081            loadIg(igs, binaries, s, false);
082          }
083        }
084      }
085      System.out.print("  Load " + src);
086      if (!src.contains("#")) {
087        System.out.print("#" + npm.version());
088      }
089      int count = getContext().loadFromPackage(npm, ValidatorUtils.loaderForVersion(npm.fhirVersion()));
090      System.out.println(" - " + count + " resources (" + getContext().clock().milestone() + ")");
091    } else {
092      System.out.print("  Load " + src);
093      String canonical = null;
094      int count = 0;
095      Map<String, byte[]> source = loadIgSource(src, recursive, true);
096      String version = Constants.VERSION;
097      if (getVersion() != null) {
098        version = getVersion();
099      }
100      if (source.containsKey("version.info")) {
101        version = readInfoVersion(source.get("version.info"));
102      }
103      for (Map.Entry<String, byte[]> t : source.entrySet()) {
104        String fn = t.getKey();
105        if (!exemptFile(fn)) {
106          Resource r = loadFileWithErrorChecking(version, t, fn);
107          if (r != null) {
108            count++;
109            getContext().cacheResource(r);
110            if (r instanceof ImplementationGuide) {
111              canonical = ((ImplementationGuide) r).getUrl();
112              igs.add((ImplementationGuide) r);
113              if (canonical.contains("/ImplementationGuide/")) {
114                Resource r2 = r.copy();
115                ((ImplementationGuide) r2).setUrl(canonical.substring(0, canonical.indexOf("/ImplementationGuide/")));
116                getContext().cacheResource(r2);
117              }
118            }
119          }
120        }
121      }
122      if (canonical != null) {
123        ValidatorUtils.grabNatives(binaries, source, canonical);
124      }
125      System.out.println(" - " + count + " resources (" + getContext().clock().milestone() + ")");
126    }
127  }
128
129  public Content loadContent(String source, String opName, boolean asIg) throws FHIRException, IOException {
130    Map<String, byte[]> s = loadIgSource(source, false, asIg);
131    Content res = new Content();
132    if (s.size() != 1)
133      throw new FHIRException("Unable to find resource " + source + " to " + opName);
134    for (Map.Entry<String, byte[]> t : s.entrySet()) {
135      res.focus = t.getValue();
136      if (t.getKey().endsWith(".json"))
137        res.cntType = Manager.FhirFormat.JSON;
138      else if (t.getKey().endsWith(".xml"))
139        res.cntType = Manager.FhirFormat.XML;
140      else if (t.getKey().endsWith(".ttl"))
141        res.cntType = Manager.FhirFormat.TURTLE;
142      else if (t.getKey().endsWith(".shc"))
143        res.cntType = Manager.FhirFormat.SHC;
144      else if (t.getKey().endsWith(".txt") || t.getKey().endsWith(".map"))
145        res.cntType = Manager.FhirFormat.TEXT;
146      else
147        throw new FHIRException("Todo: Determining resource type is not yet done");
148    }
149    return res;
150  }
151
152  /**
153   * explore should be true if we're trying to load an -ig parameter, and false if we're loading source
154   *
155   * @throws IOException
156   **/
157  public Map<String, byte[]> loadIgSource(String src,
158                                          boolean recursive,
159                                          boolean explore) throws FHIRException, IOException {
160    // src can be one of the following:
161    // - a canonical url for an ig - this will be converted to a package id and loaded into the cache
162    // - a package id for an ig - this will be loaded into the cache
163    // - a direct reference to a package ("package.tgz") - this will be extracted by the cache manager, but not put in the cache
164    // - a folder containing resources - these will be loaded directly
165    if (Common.isNetworkPath(src)) {
166      String v = null;
167      if (src.contains("|")) {
168        v = src.substring(src.indexOf("|") + 1);
169        src = src.substring(0, src.indexOf("|"));
170      }
171      String pid = explore ? getPackageCacheManager().getPackageId(src) : null;
172      if (!Utilities.noString(pid))
173        return fetchByPackage(pid + (v == null ? "" : "#" + v));
174      else
175        return fetchFromUrl(src + (v == null ? "" : "|" + v), explore);
176    }
177
178    File f = new File(Utilities.path(src));
179    if (f.exists()) {
180      if (f.isDirectory() && new File(Utilities.path(src, "package.tgz")).exists())
181        return loadPackage(new FileInputStream(Utilities.path(src, "package.tgz")), Utilities.path(src, "package.tgz"));
182      if (f.isDirectory() && new File(Utilities.path(src, "igpack.zip")).exists())
183        return readZip(new FileInputStream(Utilities.path(src, "igpack.zip")));
184      if (f.isDirectory() && new File(Utilities.path(src, "validator.pack")).exists())
185        return readZip(new FileInputStream(Utilities.path(src, "validator.pack")));
186      if (f.isDirectory())
187        return scanDirectory(f, recursive);
188      if (src.endsWith(".tgz"))
189        return loadPackage(new FileInputStream(src), src);
190      if (src.endsWith(".pack"))
191        return readZip(new FileInputStream(src));
192      if (src.endsWith("igpack.zip"))
193        return readZip(new FileInputStream(src));
194      Manager.FhirFormat fmt = ResourceChecker.checkIsResource(getContext(), isDebug(), TextFile.fileToBytes(f), src, true);
195      if (fmt != null) {
196        Map<String, byte[]> res = new HashMap<String, byte[]>();
197        res.put(Utilities.changeFileExt(src, "." + fmt.getExtension()), TextFile.fileToBytesNCS(src));
198        return res;
199      }
200    } else if ((src.matches(FilesystemPackageCacheManager.PACKAGE_REGEX) || src.matches(FilesystemPackageCacheManager.PACKAGE_VERSION_REGEX)) && !src.endsWith(".zip") && !src.endsWith(".tgz")) {
201      return fetchByPackage(src);
202    }
203    throw new FHIRException("Unable to find/resolve/read " + (explore ? "-ig " : "") + src);
204  }
205
206  public void scanForIgVersion(String src,
207                               boolean recursive,
208                               VersionSourceInformation versions) throws Exception {
209    Map<String, byte[]> source = loadIgSourceForVersion(src, recursive, true, versions);
210    if (source != null && source.containsKey("version.info"))
211      versions.see(readInfoVersion(source.get("version.info")), "version.info in " + src);
212  }
213
214  public void scanForVersions(List<String> sources, VersionSourceInformation versions) throws FHIRException, IOException {
215    List<String> refs = new ArrayList<String>();
216    ValidatorUtils.parseSources(sources, refs, context);
217    for (String ref : refs) {
218      Content cnt = loadContent(ref, "validate", false);
219      String s = TextFile.bytesToString(cnt.focus);
220      if (s.contains("http://hl7.org/fhir/3.0")) {
221        versions.see("3.0", "Profile in " + ref);
222      }
223      if (s.contains("http://hl7.org/fhir/1.0")) {
224        versions.see("1.0", "Profile in " + ref);
225      }
226      if (s.contains("http://hl7.org/fhir/4.0")) {
227        versions.see("4.0", "Profile in " + ref);
228      }
229      if (s.contains("http://hl7.org/fhir/1.4")) {
230        versions.see("1.4", "Profile in " + ref);
231      }
232      try {
233        if (s.startsWith("{")) {
234          JsonObject json = JsonTrackingParser.parse(s, null);
235          if (json.has("fhirVersion")) {
236            versions.see(VersionUtilities.getMajMin(JSONUtil.str(json, "fhirVersion")), "fhirVersion in " + ref);
237          }
238        } else {
239          Document doc = ValidatorUtils.parseXml(cnt.focus);
240          String v = XMLUtil.getNamedChildValue(doc.getDocumentElement(), "fhirVersion");
241          if (v != null) {
242            versions.see(VersionUtilities.getMajMin(v), "fhirVersion in " + ref);
243          }
244        }
245      } catch (Exception e) {
246        // nothing
247      }
248    }
249  }
250
251  protected Map<String, byte[]> readZip(InputStream stream) throws IOException {
252    Map<String, byte[]> res = new HashMap<>();
253    ZipInputStream zip = new ZipInputStream(stream);
254    ZipEntry ze;
255    while ((ze = zip.getNextEntry()) != null) {
256      String name = ze.getName();
257      ByteArrayOutputStream b = new ByteArrayOutputStream();
258      int n;
259      byte[] buf = new byte[1024];
260      while ((n = ((InputStream) zip).read(buf, 0, 1024)) > -1) {
261        b.write(buf, 0, n);
262      }
263      res.put(name, b.toByteArray());
264      zip.closeEntry();
265    }
266    zip.close();
267    return res;
268  }
269
270  private String loadPackageForVersion(InputStream stream) throws FHIRException, IOException {
271    return NpmPackage.fromPackage(stream).fhirVersion();
272  }
273
274  private InputStream fetchFromUrlSpecific(String source, boolean optional) throws FHIRException, IOException {
275    try {
276      SimpleHTTPClient http = new SimpleHTTPClient();
277      HTTPResult res = http.get(source + "?nocache=" + System.currentTimeMillis());
278      res.checkThrowException();
279      return new ByteArrayInputStream(res.getContent());
280    } catch (IOException e) {
281      if (optional)
282        return null;
283      else
284        throw e;
285    }
286  }
287
288  private Map<String, byte[]> loadIgSourceForVersion(String src,
289                                                     boolean recursive,
290                                                     boolean explore,
291                                                     VersionSourceInformation versions) throws FHIRException, IOException {
292    if (Common.isNetworkPath(src)) {
293      String v = null;
294      if (src.contains("|")) {
295        v = src.substring(src.indexOf("|") + 1);
296        src = src.substring(0, src.indexOf("|"));
297      }
298      String pid = getPackageCacheManager().getPackageId(src);
299      if (!Utilities.noString(pid)) {
300        versions.see(fetchVersionByPackage(pid + (v == null ? "" : "#" + v)), "Package " + src);
301        return null;
302      } else {
303        return fetchVersionFromUrl(src + (v == null ? "" : "|" + v), explore, versions);
304      }
305    }
306
307    File f = new File(Utilities.path(src));
308    if (f.exists()) {
309      if (f.isDirectory() && new File(Utilities.path(src, "package.tgz")).exists()) {
310        versions.see(loadPackageForVersion(new FileInputStream(Utilities.path(src, "package.tgz"))), "Package " + src);
311        return null;
312      }
313      if (f.isDirectory() && new File(Utilities.path(src, "igpack.zip")).exists())
314        return readZip(new FileInputStream(Utilities.path(src, "igpack.zip")));
315      if (f.isDirectory() && new File(Utilities.path(src, "validator.pack")).exists())
316        return readZip(new FileInputStream(Utilities.path(src, "validator.pack")));
317      if (f.isDirectory())
318        return scanDirectory(f, recursive);
319      if (src.endsWith(".tgz")) {
320        versions.see(loadPackageForVersion(new FileInputStream(src)), "Package " + src);
321        return null;
322      }
323      if (src.endsWith(".pack"))
324        return readZip(new FileInputStream(src));
325      if (src.endsWith("igpack.zip"))
326        return readZip(new FileInputStream(src));
327      Manager.FhirFormat fmt = ResourceChecker.checkIsResource(getContext(), isDebug(), TextFile.fileToBytes(f), src, true);
328      if (fmt != null) {
329        Map<String, byte[]> res = new HashMap<String, byte[]>();
330        res.put(Utilities.changeFileExt(src, "." + fmt.getExtension()), TextFile.fileToBytesNCS(src));
331        return res;
332      }
333    } else if ((src.matches(FilesystemPackageCacheManager.PACKAGE_REGEX) || src.matches(FilesystemPackageCacheManager.PACKAGE_VERSION_REGEX)) && !src.endsWith(".zip") && !src.endsWith(".tgz")) {
334      versions.see(fetchVersionByPackage(src), "Package " + src);
335      return null;
336    }
337    throw new FHIRException("Unable to find/resolve/read -ig " + src);
338  }
339
340
341  private Map<String, byte[]> fetchByPackage(String src) throws FHIRException, IOException {
342    String id = src;
343    String version = null;
344    if (src.contains("#")) {
345      id = src.substring(0, src.indexOf("#"));
346      version = src.substring(src.indexOf("#") + 1);
347    }
348    if (version == null) {
349      version = getPackageCacheManager().getLatestVersion(id);
350    }
351    NpmPackage pi;
352    if (version == null) {
353      pi = getPackageCacheManager().loadPackageFromCacheOnly(id);
354      if (pi != null)
355        System.out.println("   ... Using version " + pi.version());
356    } else
357      pi = getPackageCacheManager().loadPackageFromCacheOnly(id, version);
358    if (pi == null) {
359      return resolvePackage(id, version);
360    } else
361      return loadPackage(pi);
362  }
363
364  private Map<String, byte[]> loadPackage(InputStream stream, String name) throws FHIRException, IOException {
365    return loadPackage(NpmPackage.fromPackage(stream));
366  }
367
368  public Map<String, byte[]> loadPackage(NpmPackage pi) throws FHIRException, IOException {
369    getContext().getLoadedPackages().add(pi.name() + "#" + pi.version());
370    Map<String, byte[]> res = new HashMap<String, byte[]>();
371    for (String s : pi.dependencies()) {
372      if (s.endsWith(".x") && s.length() > 2) {
373        String packageMajorMinor = s.substring(0, s.length() - 2);
374        boolean found = false;
375        for (int i = 0; i < getContext().getLoadedPackages().size() && !found; ++i) {
376          String loadedPackage = getContext().getLoadedPackages().get(i);
377          if (loadedPackage.startsWith(packageMajorMinor)) {
378            found = true;
379          }
380        }
381        if (found)
382          continue;
383      }
384      if (!getContext().getLoadedPackages().contains(s)) {
385        if (!VersionUtilities.isCorePackage(s)) {
386          System.out.println("+  .. load IG from " + s);
387          res.putAll(fetchByPackage(s));
388        }
389      }
390    }
391
392    for (String s : pi.listResources("CodeSystem", "ConceptMap", "ImplementationGuide", "CapabilityStatement", "SearchParameter", "Conformance", "StructureMap", "ValueSet", "StructureDefinition")) {
393      res.put(s, TextFile.streamToBytes(pi.load("package", s)));
394    }
395    String ini = "[FHIR]\r\nversion=" + pi.fhirVersion() + "\r\n";
396    res.put("version.info", ini.getBytes());
397    return res;
398  }
399
400  private Map<String, byte[]> resolvePackage(String id, String v) throws FHIRException, IOException {
401    NpmPackage pi = getPackageCacheManager().loadPackage(id, v);
402    if (pi != null && v == null)
403      System.out.println("   ... Using version " + pi.version());
404    return loadPackage(pi);
405  }
406
407  private String readInfoVersion(byte[] bs) throws IOException {
408    String is = TextFile.bytesToString(bs);
409    is = is.trim();
410    IniFile ini = new IniFile(new ByteArrayInputStream(TextFile.stringToBytes(is, false)));
411    return ini.getStringProperty("FHIR", "version");
412  }
413
414  private byte[] fetchFromUrlSpecific(String source, String contentType, boolean optional, List<String> errors) throws FHIRException, IOException {
415    try {
416      SimpleHTTPClient http = new SimpleHTTPClient();
417      try {
418        // try with cache-busting option and then try withhout in case the server doesn't support that
419        HTTPResult res = http.get(source + "?nocache=" + System.currentTimeMillis(), contentType);
420        res.checkThrowException();
421        return res.getContent();
422      } catch (Exception e) {
423        HTTPResult res = http.get(source, contentType);
424        res.checkThrowException();
425        return res.getContent();
426      }
427    } catch (IOException e) {
428      if (errors != null) {
429        errors.add("Error accessing " + source + ": " + e.getMessage());
430      }
431      if (optional)
432        return null;
433      else
434        throw e;
435    }
436  }
437
438  private Map<String, byte[]> fetchVersionFromUrl(String src,
439                                                  boolean explore,
440                                                  VersionSourceInformation versions) throws FHIRException, IOException {
441    if (src.endsWith(".tgz")) {
442      versions.see(loadPackageForVersion(fetchFromUrlSpecific(src, false)), "From Package " + src);
443      return null;
444    }
445    if (src.endsWith(".pack"))
446      return readZip(fetchFromUrlSpecific(src, false));
447    if (src.endsWith("igpack.zip"))
448      return readZip(fetchFromUrlSpecific(src, false));
449
450    InputStream stream = null;
451    if (explore) {
452      stream = fetchFromUrlSpecific(Utilities.pathURL(src, "package.tgz"), true);
453      if (stream != null) {
454        versions.see(loadPackageForVersion(stream), "From Package at " + src);
455        return null;
456      }
457      // todo: these options are deprecated - remove once all IGs have been rebuilt post R4 technical correction
458      stream = fetchFromUrlSpecific(Utilities.pathURL(src, "igpack.zip"), true);
459      if (stream != null)
460        return readZip(stream);
461      stream = fetchFromUrlSpecific(Utilities.pathURL(src, "validator.pack"), true);
462      if (stream != null)
463        return readZip(stream);
464      stream = fetchFromUrlSpecific(Utilities.pathURL(src, "validator.pack"), true);
465      //// -----
466    }
467
468    // ok, having tried all that... now we'll just try to access it directly
469    byte[] cnt;
470    if (stream == null)
471      cnt = fetchFromUrlSpecific(src, "application/json", true, null);
472    else
473      cnt = TextFile.streamToBytes(stream);
474
475    Manager.FhirFormat fmt = ResourceChecker.checkIsResource(getContext(), isDebug(), cnt, src, true);
476    if (fmt != null) {
477      Map<String, byte[]> res = new HashMap<String, byte[]>();
478      res.put(Utilities.changeFileExt(src, "." + fmt.getExtension()), cnt);
479      return res;
480    }
481    String fn = Utilities.path("[tmp]", "fetch-resource-error-content.bin");
482    TextFile.bytesToFile(cnt, fn);
483    System.out.println("Error Fetching " + src);
484    System.out.println("Some content was found, saved to " + fn);
485    System.out.println("1st 100 bytes = " + presentForDebugging(cnt));
486    throw new FHIRException("Unable to find/resolve/read " + (explore ? "-ig " : "") + src);
487  }
488
489  private String fetchVersionByPackage(String src) throws FHIRException, IOException {
490    String id = src;
491    String version = null;
492    if (src.contains("#")) {
493      id = src.substring(0, src.indexOf("#"));
494      version = src.substring(src.indexOf("#") + 1);
495    }
496    if (version == null) {
497      version = getPackageCacheManager().getLatestVersion(id);
498    }
499    NpmPackage pi = null;
500    if (version == null) {
501      pi = getPackageCacheManager().loadPackageFromCacheOnly(id);
502      if (pi != null)
503        System.out.println("   ... Using version " + pi.version());
504    } else
505      pi = getPackageCacheManager().loadPackageFromCacheOnly(id, version);
506    if (pi == null) {
507      return resolvePackageForVersion(id, version);
508    } else {
509      return pi.fhirVersion();
510    }
511  }
512
513  private Map<String, byte[]> fetchFromUrl(String src, boolean explore) throws FHIRException, IOException {
514    if (src.endsWith(".tgz"))
515      return loadPackage(fetchFromUrlSpecific(src, false), src);
516    if (src.endsWith(".pack"))
517      return readZip(fetchFromUrlSpecific(src, false));
518    if (src.endsWith("igpack.zip"))
519      return readZip(fetchFromUrlSpecific(src, false));
520
521    InputStream stream = null;
522    if (explore) {
523      stream = fetchFromUrlSpecific(Utilities.pathURL(src, "package.tgz"), true);
524      if (stream != null)
525        return loadPackage(stream, Utilities.pathURL(src, "package.tgz"));
526      // todo: these options are deprecated - remove once all IGs have been rebuilt post R4 technical correction
527      stream = fetchFromUrlSpecific(Utilities.pathURL(src, "igpack.zip"), true);
528      if (stream != null)
529        return readZip(stream);
530      stream = fetchFromUrlSpecific(Utilities.pathURL(src, "validator.pack"), true);
531      if (stream != null)
532        return readZip(stream);
533      stream = fetchFromUrlSpecific(Utilities.pathURL(src, "validator.pack"), true);
534      //// -----
535    }
536
537    // ok, having tried all that... now we'll just try to access it directly
538    byte[] cnt;
539    List<String> errors = new ArrayList<>();
540    if (stream != null) {
541      cnt = TextFile.streamToBytes(stream);
542    } else {
543      cnt = fetchFromUrlSpecific(src, "application/json", true, errors);
544      if (cnt == null) {
545        cnt = fetchFromUrlSpecific(src, "application/xml", true, errors);
546      }
547    }
548    if (cnt == null) {
549      throw new FHIRException("Unable to fetch content from " + src + " (" + errors.toString() + ")");
550
551    }
552    Manager.FhirFormat fmt = checkFormat(cnt, src);
553    if (fmt != null) {
554      Map<String, byte[]> res = new HashMap<>();
555      res.put(Utilities.changeFileExt(src, "." + fmt.getExtension()), cnt);
556      return res;
557    }
558    throw new FHIRException("Unable to read content from " + src + ": cannot determine format");
559  }
560
561  private boolean isIgnoreFile(File ff) {
562    if (ff.getName().startsWith(".") || ff.getAbsolutePath().contains(".git")) {
563      return true;
564    }
565    return Utilities.existsInList(Utilities.getFileExtension(ff.getName()).toLowerCase(), IGNORED_EXTENSIONS);
566  }
567
568  private Map<String, byte[]> scanDirectory(File f, boolean recursive) throws IOException {
569    Map<String, byte[]> res = new HashMap<>();
570    for (File ff : f.listFiles()) {
571      if (ff.isDirectory() && recursive) {
572        res.putAll(scanDirectory(ff, true));
573      } else if (!ff.isDirectory() && !isIgnoreFile(ff)) {
574        Manager.FhirFormat fmt = ResourceChecker.checkIsResource(getContext(), isDebug(), TextFile.fileToBytes(ff), ff.getAbsolutePath(), true);
575        if (fmt != null) {
576          res.put(Utilities.changeFileExt(ff.getName(), "." + fmt.getExtension()), TextFile.fileToBytes(ff.getAbsolutePath()));
577        }
578      }
579    }
580    return res;
581  }
582
583  private String resolvePackageForVersion(String id, String v) throws FHIRException, IOException {
584    NpmPackage pi = getPackageCacheManager().loadPackage(id, v);
585    return pi.fhirVersion();
586  }
587
588  private String presentForDebugging(byte[] cnt) {
589    StringBuilder b = new StringBuilder();
590    for (int i = 0; i < Integer.min(cnt.length, 50); i++) {
591      b.append(Integer.toHexString(cnt[i]));
592    }
593    return b.toString();
594  }
595
596  private Manager.FhirFormat checkFormat(byte[] cnt, String filename) {
597    System.out.println("   ..Detect format for " + filename);
598    try {
599      JsonTrackingParser.parseJson(cnt);
600      return Manager.FhirFormat.JSON;
601    } catch (Exception e) {
602      log("Not JSON: " + e.getMessage());
603    }
604    try {
605      ValidatorUtils.parseXml(cnt);
606      return Manager.FhirFormat.XML;
607    } catch (Exception e) {
608      log("Not XML: " + e.getMessage());
609    }
610    try {
611      new Turtle().parse(TextFile.bytesToString(cnt));
612      return Manager.FhirFormat.TURTLE;
613    } catch (Exception e) {
614      log("Not Turtle: " + e.getMessage());
615    }
616    try {
617      new StructureMapUtilities(getContext(), null, null).parse(TextFile.bytesToString(cnt), null);
618      return Manager.FhirFormat.TEXT;
619    } catch (Exception e) {
620      log("Not Text: " + e.getMessage());
621    }
622    log("     .. not a resource: " + filename);
623    return null;
624  }
625
626  private boolean exemptFile(String fn) {
627    return Utilities.existsInList(fn, EXEMPT_FILES);
628  }
629
630  private Resource loadFileWithErrorChecking(String version, Map.Entry<String, byte[]> t, String fn) {
631    log("* load file: " + fn);
632    Resource r = null;
633    try {
634      r = loadResourceByVersion(version, t.getValue(), fn);
635      log(" .. success");
636    } catch (Exception e) {
637      if (!isDebug()) {
638        System.out.print("* load file: " + fn);
639      }
640      System.out.println(" - ignored due to error: " + (e.getMessage() == null ? " (null - NPE)" : e.getMessage()));
641      if (isDebug() || ((e.getMessage() != null && e.getMessage().contains("cannot be cast")))) {
642        e.printStackTrace();
643      }
644    }
645    return r;
646  }
647
648  public Resource loadResourceByVersion(String fhirVersion, byte[] content, String fn) throws IOException, FHIRException {
649    Resource r;
650    if (fhirVersion.startsWith("3.0")) {
651      org.hl7.fhir.dstu3.model.Resource res;
652      if (fn.endsWith(".xml") && !fn.endsWith("template.xml"))
653        res = new org.hl7.fhir.dstu3.formats.XmlParser().parse(new ByteArrayInputStream(content));
654      else if (fn.endsWith(".json") && !fn.endsWith("template.json"))
655        res = new org.hl7.fhir.dstu3.formats.JsonParser().parse(new ByteArrayInputStream(content));
656      else if (fn.endsWith(".txt") || fn.endsWith(".map"))
657        res = new org.hl7.fhir.dstu3.utils.StructureMapUtilities(null).parse(new String(content));
658      else
659        throw new FHIRException("Unsupported format for " + fn);
660      r = VersionConvertorFactory_30_50.convertResource(res);
661    } else if (fhirVersion.startsWith("4.0")) {
662      org.hl7.fhir.r4.model.Resource res;
663      if (fn.endsWith(".xml") && !fn.endsWith("template.xml"))
664        res = new org.hl7.fhir.r4.formats.XmlParser().parse(new ByteArrayInputStream(content));
665      else if (fn.endsWith(".json") && !fn.endsWith("template.json"))
666        res = new org.hl7.fhir.r4.formats.JsonParser().parse(new ByteArrayInputStream(content));
667      else if (fn.endsWith(".txt") || fn.endsWith(".map"))
668        res = new org.hl7.fhir.r4.utils.StructureMapUtilities(null).parse(new String(content), fn);
669      else
670        throw new FHIRException("Unsupported format for " + fn);
671      r = VersionConvertorFactory_40_50.convertResource(res);
672    } else if (fhirVersion.startsWith("1.4")) {
673      org.hl7.fhir.dstu2016may.model.Resource res;
674      if (fn.endsWith(".xml") && !fn.endsWith("template.xml"))
675        res = new org.hl7.fhir.dstu2016may.formats.XmlParser().parse(new ByteArrayInputStream(content));
676      else if (fn.endsWith(".json") && !fn.endsWith("template.json"))
677        res = new org.hl7.fhir.dstu2016may.formats.JsonParser().parse(new ByteArrayInputStream(content));
678      else
679        throw new FHIRException("Unsupported format for " + fn);
680      r = VersionConvertorFactory_14_50.convertResource(res);
681    } else if (fhirVersion.startsWith("1.0")) {
682      org.hl7.fhir.dstu2.model.Resource res;
683      if (fn.endsWith(".xml") && !fn.endsWith("template.xml"))
684        res = new org.hl7.fhir.dstu2.formats.JsonParser().parse(new ByteArrayInputStream(content));
685      else if (fn.endsWith(".json") && !fn.endsWith("template.json"))
686        res = new org.hl7.fhir.dstu2.formats.JsonParser().parse(new ByteArrayInputStream(content));
687      else
688        throw new FHIRException("Unsupported format for " + fn);
689      r = VersionConvertorFactory_10_50.convertResource(res, new org.hl7.fhir.convertors.misc.IGR2ConvertorAdvisor5());
690    } else if (fhirVersion.equals(Constants.VERSION) || "current".equals(fhirVersion)) {
691      if (fn.endsWith(".xml") && !fn.endsWith("template.xml"))
692        r = new XmlParser().parse(new ByteArrayInputStream(content));
693      else if (fn.endsWith(".json") && !fn.endsWith("template.json"))
694        r = new JsonParser().parse(new ByteArrayInputStream(content));
695      else if (fn.endsWith(".txt"))
696        r = new StructureMapUtilities(getContext(), null, null).parse(TextFile.bytesToString(content), fn);
697      else if (fn.endsWith(".map"))
698        r = new StructureMapUtilities(null).parse(new String(content), fn);
699      else
700        throw new FHIRException("Unsupported format for " + fn);
701    } else
702      throw new FHIRException("Unsupported version " + fhirVersion);
703    return r;
704  }
705
706  private void log(String s) {
707    if (isDebug()) System.out.println(s);
708  }
709}