Source code

001package org.hl7.fhir.utilities.npm;
002
003/*
004  Copyright (c) 2011+, HL7, Inc.
005  All rights reserved.
006  
007  Redistribution and use in source and binary forms, with or without modification, 
008  are permitted provided that the following conditions are met:
009    
010   * Redistributions of source code must retain the above copyright notice, this 
011     list of conditions and the following disclaimer.
012   * Redistributions in binary form must reproduce the above copyright notice, 
013     this list of conditions and the following disclaimer in the documentation 
014     and/or other materials provided with the distribution.
015   * Neither the name of HL7 nor the names of its contributors may be used to 
016     endorse or promote products derived from this software without specific 
017     prior written permission.
018  
019  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
020  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
021  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 
022  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
023  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 
024  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
025  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
026  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
027  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
028  POSSIBILITY OF SUCH DAMAGE.
029  
030 */
031
032
033
034import java.io.BufferedOutputStream;
035import java.io.ByteArrayInputStream;
036import java.io.ByteArrayOutputStream;
037import java.io.File;
038import java.io.FileInputStream;
039import java.io.FileNotFoundException;
040import java.io.IOException;
041import java.io.InputStream;
042import java.io.OutputStream;
043import java.nio.charset.Charset;
044import java.nio.charset.StandardCharsets;
045import java.util.ArrayList;
046import java.util.Collections;
047import java.util.Comparator;
048import java.util.HashMap;
049import java.util.List;
050import java.util.Map;
051import java.util.Map.Entry;
052import java.util.Set;
053import java.util.zip.ZipEntry;
054import java.util.zip.ZipInputStream;
055
056import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
057import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
058import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
059import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
060import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
061import org.hl7.fhir.exceptions.FHIRException;
062import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
063import org.hl7.fhir.utilities.TextFile;
064import org.hl7.fhir.utilities.Utilities;
065import org.hl7.fhir.utilities.json.JSONUtil;
066import org.hl7.fhir.utilities.json.JsonTrackingParser;
067import org.hl7.fhir.utilities.npm.NpmPackage.ITransformingLoader;
068import org.hl7.fhir.utilities.npm.NpmPackage.PackageResourceInformationSorter;
069import org.hl7.fhir.utilities.npm.PackageGenerator.PackageType;
070
071import com.google.gson.GsonBuilder;
072import com.google.gson.JsonArray;
073import com.google.gson.JsonElement;
074import com.google.gson.JsonObject;
075
076/**
077 * info and loader for a package 
078 * 
079 * Packages may exist on disk in the cache, or purely in memory when they are loaded on the fly
080 * 
081 * Packages are contained in subfolders (see the package spec). The FHIR resources will be in "package"
082 * 
083 * @author Grahame Grieve
084 *
085 */
086public class NpmPackage {
087
088  public interface ITransformingLoader {
089
090    byte[] load(File f);
091
092  }
093
094  public class PackageResourceInformationSorter implements Comparator<PackageResourceInformation> {
095    @Override
096    public int compare(PackageResourceInformation o1, PackageResourceInformation o2) {
097      return o1.filename.compareTo(o2.filename);
098    }
099  }
100  
101  public class PackageResourceInformation {
102    private String id;
103    private String type;
104    private String url;
105    private String version;
106    private String filename;
107    private String supplements;
108    
109    public PackageResourceInformation(String root, JsonObject fi) throws IOException {
110      super();
111      id = JSONUtil.str(fi, "id");
112      type = JSONUtil.str(fi, "resourceType");
113      url = JSONUtil.str(fi, "url");
114      version = JSONUtil.str(fi, "version");
115      filename = Utilities.path(root, JSONUtil.str(fi, "filename"));
116      supplements = JSONUtil.str(fi, "supplements");
117    }
118    public String getId() {
119      return id;
120    }
121    public String getType() {
122      return type;
123    }
124    public String getUrl() {
125      return url;
126    }
127    public String getVersion() {
128      return version;
129    }
130    public String getFilename() {
131      return filename;
132    }
133    public String getSupplements() {
134      return supplements;
135    }
136    
137  }
138  public class IndexVersionSorter implements Comparator<JsonObject> {
139
140    @Override
141    public int compare(JsonObject o0, JsonObject o1) {
142      String v0 = JSONUtil.str(o0, "version"); 
143      String v1 = JSONUtil.str(o1, "version"); 
144      return v0.compareTo(v1);
145    }
146  }
147
148  public static boolean isValidName(String pid) {
149    return pid.matches("^[a-z][a-zA-Z0-9]*(\\.[a-z][a-zA-Z0-9\\-]*)+$");
150  }
151
152  public static boolean isValidVersion(String ver) {
153    return ver.matches("^[0-9]+\\.[0-9]+\\.[0-9]+$");
154  }
155
156  public class NpmPackageFolder {
157    private String name;
158    private Map<String, List<String>> types = new HashMap<>();
159    private Map<String, byte[]> content = new HashMap<>();
160    private JsonObject index;
161    private File folder;
162
163    public NpmPackageFolder(String name) {
164      super();
165      this.name = name;
166    }
167
168    public Map<String, List<String>> getTypes() {
169      return types;
170    }
171
172    public String getName() {
173      return name;
174    }
175
176    public boolean readIndex(JsonObject index) {
177      if (!index.has("index-version") || (index.get("index-version").getAsInt() != 1)) {
178        return false;
179      }
180      this.index = index;
181      for (JsonElement e : index.getAsJsonArray("files")) {
182        JsonObject file = (JsonObject) e;
183        String type = JSONUtil.str(file, "resourceType");
184        String name = JSONUtil.str(file, "filename");
185        if (!types.containsKey(type))
186          types.put(type, new ArrayList<>());
187        types.get(type).add(name);
188      }
189      return true;
190    }
191
192    public List<String> listFiles() {
193      List<String> res = new ArrayList<>();
194      if (folder != null) {
195        for (File f : folder.listFiles()) {
196          if (!f.isDirectory() && !Utilities.existsInList(f.getName(), "package.json", ".index.json")) {
197            res.add(f.getName());
198          }
199        }
200      } else {
201        for (String s : content.keySet()) {
202          if (!Utilities.existsInList(s, "package.json", ".index.json")) {
203            res.add(s);
204          }
205        }
206      }
207      Collections.sort(res);
208      return res;
209    }
210
211    public Map<String, byte[]> getContent() {
212      return content;
213    }
214
215    public byte[] fetchFile(String file) throws FileNotFoundException, IOException {
216      if (folder != null) {
217        File f = new File(Utilities.path(folder.getAbsolutePath(), file));
218        if (f.exists()) {
219          return TextFile.fileToBytes(f);
220        } else {
221          return null;
222        }
223      } else {
224        return content.get(file);
225      }
226    }
227
228    public boolean hasFile(String file) throws IOException {
229      if (folder != null) {
230        return new File(Utilities.path(folder.getAbsolutePath(), file)).exists();
231      } else {
232        return content.containsKey(file);
233      }
234
235    }
236
237    public String dump() {
238      return name + " ("+ (folder == null ? "null" : folder.toString())+") | "+Boolean.toString(index != null)+" | "+content.size()+" | "+types.size();
239    }
240
241    public void removeFile(String n) throws IOException {
242      if (folder != null) {
243        new File(Utilities.path(folder.getAbsolutePath(), n)).delete();
244      } else {
245        content.remove(n);
246      }
247      changedByLoader = true;      
248    }
249
250  }
251
252  private String path;
253  private JsonObject npm;
254  private Map<String, NpmPackageFolder> folders = new HashMap<>();
255  private boolean changedByLoader; // internal qa only!
256  private Map<String, Object> userData = new HashMap<>();
257
258  /**
259   * Constructor
260   */
261  private NpmPackage() {
262    super();
263  }
264
265  /**
266   * Factory method that parses a package from an extracted folder
267   */
268  public static NpmPackage fromFolder(String path) throws IOException {
269    NpmPackage res = new NpmPackage();
270    res.loadFiles(path, new File(path));
271    res.checkIndexed(path);
272    return res;
273  }
274
275  /**
276   * Factory method that starts a new empty package using the given PackageGenerator to create the manifest
277   */
278  public static NpmPackage empty(PackageGenerator thePackageGenerator) {
279    NpmPackage retVal = new NpmPackage();
280    retVal.npm = thePackageGenerator.getRootJsonObject();
281    return retVal;
282  }
283
284  public Map<String, Object> getUserData() {
285    return userData;
286  }
287
288  public void loadFiles(String path, File source, String... exemptions) throws FileNotFoundException, IOException {
289    this.npm = (JsonObject) new com.google.gson.JsonParser().parse(TextFile.fileToString(Utilities.path(path, "package", "package.json")));
290    this.path = path;
291    
292    File dir = new File(path);
293    for (File f : dir.listFiles()) {
294      if (!isInternalExemptFile(f) && !Utilities.existsInList(f.getName(), exemptions)) {
295        if (f.isDirectory()) {
296          String d = f.getName();
297          if (!d.equals("package")) {
298            d = Utilities.path("package", d);
299          }
300          NpmPackageFolder folder = this.new NpmPackageFolder(d);
301          folder.folder = f;
302          this.folders.put(d, folder);
303          File ij = new File(Utilities.path(f.getAbsolutePath(), ".index.json"));
304          if (ij.exists()) {
305            try {
306              if (!folder.readIndex(JsonTrackingParser.parseJson(ij))) {
307                indexFolder(folder.getName(), folder);
308              }
309            } catch (Exception e) {
310              throw new IOException("Error parsing "+ij.getAbsolutePath()+": "+e.getMessage(), e);
311            }
312          }
313          loadSubFolders(dir.getAbsolutePath(), f);
314        } else {
315          NpmPackageFolder folder = this.new NpmPackageFolder(Utilities.path("package", "$root"));
316          folder.folder = dir;
317          this.folders.put(Utilities.path("package", "$root"), folder);        
318        }
319      }
320    }
321  }
322
323  public static boolean isInternalExemptFile(File f) {
324    return Utilities.existsInList(f.getName(), ".git", ".svn") || Utilities.existsInList(f.getName(), "package-list.json");
325  }
326
327  private void loadSubFolders(String rootPath, File dir) throws IOException {
328    for (File f : dir.listFiles()) {
329      if (f.isDirectory()) {
330        String d = f.getAbsolutePath().substring(rootPath.length()+1);
331        if (!d.startsWith("package")) {
332          d = Utilities.path("package", d);
333        }
334        NpmPackageFolder folder = this.new NpmPackageFolder(d);
335        folder.folder = f;
336        this.folders.put(d, folder);
337        File ij = new File(Utilities.path(f.getAbsolutePath(), ".index.json"));
338        if (ij.exists()) {
339          try {
340            if (!folder.readIndex(JsonTrackingParser.parseJson(ij))) {
341              indexFolder(folder.getName(), folder);
342            }
343          } catch (Exception e) {
344            throw new IOException("Error parsing "+ij.getAbsolutePath()+": "+e.getMessage(), e);
345          }
346        }
347        loadSubFolders(rootPath, f);        
348      }
349    }    
350  }
351
352  public static NpmPackage fromFolder(String folder, PackageType defType, String... exemptions) throws IOException {
353    NpmPackage res = new NpmPackage();
354    res.loadFiles(folder, new File(folder), exemptions);
355    if (!res.folders.containsKey("package")) {
356      res.folders.put("package", res.new NpmPackageFolder("package"));
357    }
358    if (!res.folders.get("package").hasFile("package.json") && defType != null) {
359      TextFile.stringToFile("{ \"type\" : \""+defType.getCode()+"\"}", Utilities.path(res.folders.get("package").folder.getAbsolutePath(), "package.json"));
360    }
361    res.npm = (JsonObject) new com.google.gson.JsonParser().parse(new String(res.folders.get("package").fetchFile("package.json")));
362    return res;
363  }
364
365  private static final int BUFFER_SIZE = 1024;
366
367  public static NpmPackage fromPackage(InputStream tgz) throws IOException {
368    return fromPackage(tgz, null, false);
369  }
370
371  public static NpmPackage fromPackage(InputStream tgz, String desc) throws IOException {
372    return fromPackage(tgz, desc, false);
373  }
374
375  public static NpmPackage fromPackage(InputStream tgz, String desc, boolean progress) throws IOException {
376    NpmPackage res = new NpmPackage();
377    res.readStream(tgz, desc, progress);
378    return res;
379  }
380
381  public void readStream(InputStream tgz, String desc, boolean progress) throws IOException {
382    GzipCompressorInputStream gzipIn;
383    try {
384      gzipIn = new GzipCompressorInputStream(tgz);
385    } catch (Exception e) {
386      throw new IOException("Error reading "+(desc == null ? "package" : desc)+": "+e.getMessage(), e);      
387    }
388    try (TarArchiveInputStream tarIn = new TarArchiveInputStream(gzipIn)) {
389      TarArchiveEntry entry;
390
391      int i = 0;
392      int c = 12;
393      while ((entry = (TarArchiveEntry) tarIn.getNextEntry()) != null) {
394        i++;
395        String n = entry.getName();
396        if (entry.isDirectory()) {
397          String dir = n.substring(0, n.length()-1);
398          if (dir.startsWith("package/")) {
399            dir = dir.substring(8);
400          }
401          folders.put(dir, new NpmPackageFolder(dir));
402        } else {
403          int count;
404          byte data[] = new byte[BUFFER_SIZE];
405          ByteArrayOutputStream fos = new ByteArrayOutputStream();
406          try (BufferedOutputStream dest = new BufferedOutputStream(fos, BUFFER_SIZE)) {
407            while ((count = tarIn.read(data, 0, BUFFER_SIZE)) != -1) {
408              dest.write(data, 0, count);
409            }
410          }
411          fos.close();
412          loadFile(n, fos.toByteArray());
413        }
414        if (progress && i % 50 == 0) {
415          c++;
416          System.out.print(".");
417          if (c == 120) {
418            System.out.println("");
419            System.out.print("  ");
420            c = 2;
421          }
422        }
423      }
424    } 
425    try {
426      npm = JsonTrackingParser.parseJson(folders.get("package").fetchFile("package.json"));
427    } catch (Exception e) {
428      throw new IOException("Error parsing "+(desc == null ? "" : desc+"#")+"package/package.json: "+e.getMessage(), e);
429    }
430    checkIndexed(desc);
431  }
432
433  public void loadFile(String n, byte[] data) throws IOException {
434    String dir = n.contains("/") ? n.substring(0, n.lastIndexOf("/")) : "$root";
435    if (dir.startsWith("package/")) {
436      dir = dir.substring(8);
437    }
438    n = n.substring(n.lastIndexOf("/")+1);
439    NpmPackageFolder index = folders.get(dir);
440    if (index == null) {
441      index = new NpmPackageFolder(dir);
442      folders.put(dir, index);
443    }
444    index.content.put(n, data);
445  }
446
447  private void checkIndexed(String desc) throws IOException {
448    for (NpmPackageFolder folder : folders.values()) {
449      if (folder.index == null) {
450        indexFolder(desc, folder);
451      }
452    }
453  }
454
455  public void indexFolder(String desc, NpmPackageFolder folder) throws FileNotFoundException, IOException {
456    List<String> remove = new ArrayList<>();
457    NpmPackageIndexBuilder indexer = new NpmPackageIndexBuilder();
458    indexer.start();
459    for (String n : folder.listFiles()) {
460      if (!indexer.seeFile(n, folder.fetchFile(n))) {
461        remove.add(n);
462      }
463    } 
464    for (String n : remove) {
465      folder.removeFile(n);
466    }
467    String json = indexer.build();
468    try {
469      folder.readIndex(JsonTrackingParser.parseJson(json));
470      if (folder.folder != null) {
471        TextFile.stringToFile(json, Utilities.path(folder.folder.getAbsolutePath(), ".index.json"));
472      }
473    } catch (Exception e) {
474      TextFile.stringToFile(json, Utilities.path("[tmp]", ".index.json"));
475      throw new IOException("Error parsing "+(desc == null ? "" : desc+"#")+"package/"+folder.name+"/.index.json: "+e.getMessage(), e);
476    }
477  }
478
479
480  public static NpmPackage fromZip(InputStream stream, boolean dropRootFolder, String desc) throws IOException {
481    NpmPackage res = new NpmPackage();
482    ZipInputStream zip = new ZipInputStream(stream);
483    ZipEntry ze;
484    while ((ze = zip.getNextEntry()) != null) {
485      int size;
486      byte[] buffer = new byte[2048];
487
488      ByteArrayOutputStream bytes = new ByteArrayOutputStream();
489      BufferedOutputStream bos = new BufferedOutputStream(bytes, buffer.length);
490
491      while ((size = zip.read(buffer, 0, buffer.length)) != -1) {
492        bos.write(buffer, 0, size);
493      }
494      bos.flush();
495      bos.close();
496      if (bytes.size() > 0) {
497        if (dropRootFolder) {
498          res.loadFile(ze.getName().substring(ze.getName().indexOf("/")+1), bytes.toByteArray());
499        } else {
500          res.loadFile(ze.getName(), bytes.toByteArray());
501        }
502      }
503      zip.closeEntry();
504    }
505    zip.close();         
506    try {
507      res.npm = JsonTrackingParser.parseJson(res.folders.get("package").fetchFile("package.json"));
508    } catch (Exception e) {
509      throw new IOException("Error parsing "+(desc == null ? "" : desc+"#")+"package/package.json: "+e.getMessage(), e);
510    }
511    res.checkIndexed(desc);
512    return res;
513  }
514
515
516  /**
517   * Accessing the contents of the package - get a list of files in a subfolder of the package 
518   *
519   * @param folder
520   * @return
521   * @throws IOException 
522   */
523  public List<String> list(String folder) throws IOException {
524    List<String> res = new ArrayList<String>();
525    if (folders.containsKey(folder)) {
526      res.addAll(folders.get(folder).listFiles());
527    } else if (folders.containsKey(Utilities.path("package", folder))) {
528      res.addAll(folders.get(Utilities.path("package", folder)).listFiles());
529    }
530    return res;
531  }
532
533  public List<String> listResources(String... types) throws IOException {
534    List<String> res = new ArrayList<String>();
535    NpmPackageFolder folder = folders.get("package");
536    for (String s : types) {
537      if (folder.types.containsKey(s))
538        res.addAll(folder.types.get(s));
539    }
540    Collections.sort(res);
541    return res;
542  }
543
544  public List<PackageResourceInformation> listIndexedResources(String... types) throws IOException {
545    List<PackageResourceInformation> res = new ArrayList<PackageResourceInformation>();
546    for (NpmPackageFolder folder : folders.values()) {
547      if (folder.index != null) {
548        for (JsonElement e : folder.index.getAsJsonArray("files")) {
549          JsonObject fi = e.getAsJsonObject();
550          if (Utilities.existsInList(JSONUtil.str(fi, "resourceType"), types)) {
551            res.add(new PackageResourceInformation(folder.folder.getAbsolutePath(), fi));
552          }
553        }
554      }
555    } 
556    //    Collections.sort(res, new PackageResourceInformationSorter());
557    return res;
558  }
559
560  /**
561   * use the name from listResources()
562   * 
563   * @param id
564   * @return
565   * @throws IOException
566   */
567  public InputStream loadResource(String file) throws IOException {
568    NpmPackageFolder folder = folders.get("package");
569    return new ByteArrayInputStream(folder.fetchFile(file));
570  }
571
572  /**
573   * get a stream that contains the contents of a resource in the base folder, by it's canonical URL
574   * 
575   * @param url - the canonical URL of the resource (exact match only)
576   * @return null if it is not found
577   * @throws IOException
578   */
579  public InputStream loadByCanonical(String canonical) throws IOException {
580    return loadByCanonicalVersion("package", canonical, null);    
581  }
582  
583  /**
584   * get a stream that contains the contents of a resource in the nominated folder, by it's canonical URL
585   * 
586   * @param folder - one of the folders in the package (main folder is "package")
587   * @param url - the canonical URL of the resource (exact match only)
588   * @return null if it is not found
589   * @throws IOException
590   */
591  public InputStream loadByCanonical(String folder, String canonical) throws IOException {
592    return loadByCanonicalVersion(folder, canonical, null);    
593  }
594    
595  /**
596   * get a stream that contains the contents of a resource in the base folder, by it's canonical URL
597   * 
598   * @param url - the canonical URL of the resource (exact match only)
599   * @param version - the specified version (or null if the most recent)
600   * 
601   * @return null if it is not found
602   * @throws IOException
603   */
604  public InputStream loadByCanonicalVersion(String canonical, String version) throws IOException {
605    return loadByCanonicalVersion("package", canonical, version);
606  }
607  
608  /**
609   * get a stream that contains the contents of a resource in the nominated folder, by it's canonical URL
610   * 
611   * @param folder - one of the folders in the package (main folder is "package")
612   * @param url - the canonical URL of the resource (exact match only)
613   * @param version - the specified version (or null if the most recent)
614   * 
615   * @return null if it is not found
616   * @throws IOException
617   */
618  public InputStream loadByCanonicalVersion(String folder, String canonical, String version) throws IOException {
619    NpmPackageFolder f = folders.get(folder);
620    List<JsonObject> matches = new ArrayList<>();
621    for (JsonElement e : f.index.getAsJsonArray("files")) {
622      JsonObject file = (JsonObject) e;
623      if (canonical.equals(JSONUtil.str(file, "url"))) {
624        if (version != null && version.equals(JSONUtil.str(file, "version"))) {
625          return load("package", JSONUtil.str(file, "filename"));
626        } else if (version == null) {
627          matches.add(file);
628        }
629      }
630      if (matches.size() > 0) {
631        if (matches.size() == 1) {
632          return load("package", JSONUtil.str(matches.get(0), "filename"));          
633        } else {
634          Collections.sort(matches, new IndexVersionSorter());
635          return load("package", JSONUtil.str(matches.get(matches.size()-1), "filename"));          
636        }
637      }
638    }
639    return null;        
640  }
641    
642  /**
643   * get a stream that contains the contents of one of the files in the base package
644   * 
645   * @param file
646   * @return
647   * @throws IOException
648   */
649  public InputStream load(String file) throws IOException {
650    return load("package", file);
651  }
652  /**
653   * get a stream that contains the contents of one of the files in a folder
654   * 
655   * @param folder
656   * @param file
657   * @return
658   * @throws IOException
659   */
660  public InputStream load(String folder, String file) throws IOException {
661    NpmPackageFolder f = folders.get(folder);
662    if (f == null) {
663      f = folders.get(Utilities.path("package", folder));
664    }
665    if (f != null && f.hasFile(file)) {
666      return new ByteArrayInputStream(f.fetchFile(file));
667    } else {
668      throw new IOException("Unable to find the file "+folder+"/"+file+" in the package "+name());
669    }
670  }
671
672  public boolean hasFile(String folder, String file) throws IOException {
673    NpmPackageFolder f = folders.get(folder);
674    if (f == null) {
675      f = folders.get(Utilities.path("package", folder));
676    }
677    return f != null && f.hasFile(file);
678  }
679
680
681  /**
682   * Handle to the package json file
683   * 
684   * @return
685   */
686  public JsonObject getNpm() {
687    return npm;
688  }
689
690  /**
691   * convenience method for getting the package name
692   * @return
693   */
694  public String name() {
695    return JSONUtil.str(npm, "name");
696  }
697
698  /**
699   * convenience method for getting the package id (which in NPM language is the same as the name)
700   * @return
701   */
702  public String id() {
703    return JSONUtil.str(npm, "name");
704  }
705
706  public String date() {
707    return JSONUtil.str(npm, "date");
708  }
709
710  public String canonical() {
711    return JSONUtil.str(npm, "canonical");
712  }
713
714  /**
715   * convenience method for getting the package version
716   * @return
717   */
718  public String version() {
719    return JSONUtil.str(npm, "version");
720  }
721
722  /**
723   * convenience method for getting the package fhir version
724   * @return
725   */
726  public String fhirVersion() {
727    if ("hl7.fhir.core".equals(JSONUtil.str(npm, "name")))
728      return JSONUtil.str(npm, "version");
729    else if (JSONUtil.str(npm, "name").startsWith("hl7.fhir.r2.") || JSONUtil.str(npm, "name").startsWith("hl7.fhir.r2b.") || JSONUtil.str(npm, "name").startsWith("hl7.fhir.r3.") || 
730        JSONUtil.str(npm, "name").startsWith("hl7.fhir.r4.") || JSONUtil.str(npm, "name").startsWith("hl7.fhir.r4b.") || JSONUtil.str(npm, "name").startsWith("hl7.fhir.r5."))
731      return JSONUtil.str(npm, "version");
732    else {
733      JsonObject dep = null;
734      if (npm.has("dependencies") && npm.get("dependencies").isJsonObject()) {
735        dep = npm.getAsJsonObject("dependencies");
736        if (dep != null) {
737          for (Entry<String, JsonElement> e : dep.entrySet()) {
738            if (Utilities.existsInList(e.getKey(), "hl7.fhir.r2.core", "hl7.fhir.r2b.core", "hl7.fhir.r3.core", "hl7.fhir.r4.core"))
739              return e.getValue().getAsString();
740            if (Utilities.existsInList(e.getKey(), "hl7.fhir.core")) // while all packages are updated
741              return e.getValue().getAsString();
742          }
743        }
744      }
745      if (npm.has("fhirVersions")) {
746        JsonElement e = npm.get("fhirVersions");
747        if (e.isJsonArray() && e.getAsJsonArray().size() > 0) {
748          return npm.getAsJsonArray("fhirVersions").get(0).getAsString();
749        }
750      }
751      if (dep != null) {
752        // legacy simplifier support:
753        if (dep.has("simplifier.core.r4"))
754          return "4.0";
755        if (dep.has("simplifier.core.r3"))
756          return "3.0";
757        if (dep.has("simplifier.core.r2"))
758          return "2.0";
759      }
760      throw new FHIRException("no core dependency or FHIR Version found in the Package definition");
761    }
762  }
763
764  public String summary() {
765    if (path != null)
766      return path;
767    else
768      return "memory";
769  }
770
771  public boolean isType(PackageType template) {
772    return template.getCode().equals(type());
773  }
774
775  public String type() {
776    return JSONUtil.str(npm, "type");
777  }
778
779  public String description() {
780    return JSONUtil.str(npm, "description");
781  }
782
783  public String getPath() {
784    return path;
785  }
786
787  public List<String> dependencies() {
788    List<String> res = new ArrayList<>();
789    if (npm.has("dependencies")) {
790      for (Entry<String, JsonElement> e : npm.getAsJsonObject("dependencies").entrySet()) {
791        res.add(e.getKey()+"#"+e.getValue().getAsString());
792      }
793    }
794    return res;
795  }
796
797  public String homepage() {
798    return JSONUtil.str(npm, "homepage");
799  }
800
801  public String url() {
802    return JSONUtil.str(npm, "url");
803  }
804
805
806  public String title() {
807    return JSONUtil.str(npm, "title");
808  }
809
810  public String toolsVersion() {
811    return JSONUtil.str(npm, "tools-version");
812  }
813
814  public String license() {
815    return JSONUtil.str(npm, "license");
816  }
817
818  //  /**
819  //   * only for use by the package manager itself
820  //   * 
821  //   * @param path
822  //   */
823  //  public void setPath(String path) {
824  //    this.path = path;
825  //  }
826
827  public String getWebLocation() {
828    if (npm.has("url") && npm.get("url").isJsonPrimitive()) {
829      return PackageHacker.fixPackageUrl(npm.get("url").getAsString());
830    } else {
831      return JSONUtil.str(npm, "canonical");
832    }
833  }
834
835  public InputStream loadResource(String type, String id) throws IOException {
836    NpmPackageFolder f = folders.get("package");
837    JsonArray files = f.index.getAsJsonArray("files");
838    for (JsonElement e : files) {
839      JsonObject i = (JsonObject) e;
840      if (type.equals(JSONUtil.str(i, "resourceType")) && id.equals(JSONUtil.str(i, "id"))) {
841        return load("package", JSONUtil.str(i, "filename"));
842      }
843    }
844    return null;
845  }
846
847  public InputStream loadExampleResource(String type, String id) throws IOException {
848    NpmPackageFolder f = folders.get("example");
849    if (f != null) {
850      JsonArray files = f.index.getAsJsonArray("files");
851      for (JsonElement e : files) {
852        JsonObject i = (JsonObject) e;
853        if (type.equals(JSONUtil.str(i, "resourceType")) && id.equals(JSONUtil.str(i, "id"))) {
854          return load("example", JSONUtil.str(i, "filename"));
855        }
856      }
857    }
858    return null;
859  }
860
861  /** special case when playing around inside the package **/
862  public Map<String, NpmPackageFolder> getFolders() {
863    return folders;
864  }
865
866  public void save(File directory) throws IOException {
867    File dir = new File(Utilities.path(directory.getAbsolutePath(), name()));
868    if (!dir.exists()) {
869      Utilities.createDirectory(dir.getAbsolutePath());
870    } else {
871      Utilities.clearDirectory(dir.getAbsolutePath());
872    }
873    
874    for (NpmPackageFolder folder : folders.values()) {
875      String n = folder.name;
876
877      File pd = new File(Utilities.path(dir.getAbsolutePath(), n));
878      if (!pd.exists()) {
879        Utilities.createDirectory(pd.getAbsolutePath());
880      }
881      NpmPackageIndexBuilder indexer = new NpmPackageIndexBuilder();
882      indexer.start();
883      for (String s : folder.content.keySet()) {
884        byte[] b = folder.content.get(s);
885        indexer.seeFile(s, b);
886        if (!s.equals(".index.json") && !s.equals("package.json")) {
887          TextFile.bytesToFile(b, Utilities.path(dir.getAbsolutePath(), n, s));
888        }
889      }
890      byte[] cnt = indexer.build().getBytes(StandardCharsets.UTF_8);
891      TextFile.bytesToFile(cnt, Utilities.path(dir.getAbsolutePath(), n, ".index.json"));
892    }
893    byte[] cnt = TextFile.stringToBytes(new GsonBuilder().setPrettyPrinting().create().toJson(npm), false);
894    TextFile.bytesToFile(cnt, Utilities.path(dir.getAbsolutePath(), "package", "package.json"));
895  }
896  
897  public void save(OutputStream stream) throws IOException {
898    TarArchiveOutputStream tar;
899    ByteArrayOutputStream OutputStream;
900    BufferedOutputStream bufferedOutputStream;
901    GzipCompressorOutputStream gzipOutputStream;
902
903    OutputStream = new ByteArrayOutputStream();
904    bufferedOutputStream = new BufferedOutputStream(OutputStream);
905    gzipOutputStream = new GzipCompressorOutputStream(bufferedOutputStream);
906    tar = new TarArchiveOutputStream(gzipOutputStream);
907
908
909    for (NpmPackageFolder folder : folders.values()) {
910      String n = folder.name;
911      if (!"package".equals(n) && !(n.startsWith("package/") || n.startsWith("package\\"))) {
912        n = "package/"+n;
913      }
914      NpmPackageIndexBuilder indexer = new NpmPackageIndexBuilder();
915      indexer.start();
916      for (String s : folder.content.keySet()) {
917        byte[] b = folder.content.get(s);
918        String name = n+"/"+s;
919        indexer.seeFile(s, b);
920        if (!s.equals(".index.json") && !s.equals("package.json")) {
921          TarArchiveEntry entry = new TarArchiveEntry(name);
922          entry.setSize(b.length);
923          tar.putArchiveEntry(entry);
924          tar.write(b);
925          tar.closeArchiveEntry();
926        }
927      }
928      byte[] cnt = indexer.build().getBytes(StandardCharsets.UTF_8);
929      TarArchiveEntry entry = new TarArchiveEntry(n+"/.index.json");
930      entry.setSize(cnt.length);
931      tar.putArchiveEntry(entry);
932      tar.write(cnt);
933      tar.closeArchiveEntry();
934    }
935    byte[] cnt = TextFile.stringToBytes(new GsonBuilder().setPrettyPrinting().create().toJson(npm), false);
936    TarArchiveEntry entry = new TarArchiveEntry("package/package.json");
937    entry.setSize(cnt.length);
938    tar.putArchiveEntry(entry);
939    tar.write(cnt);
940    tar.closeArchiveEntry();
941
942    tar.finish();
943    tar.close();
944    gzipOutputStream.close();
945    bufferedOutputStream.close();
946    OutputStream.close();
947    byte[] b = OutputStream.toByteArray();
948    stream.write(b);
949  }
950
951  /**
952   * Keys are resource type names, values are filenames
953   */
954  public Map<String, List<String>> getTypes() {
955    return folders.get("package").types;
956  }
957
958  public String fhirVersionList() {
959    if (npm.has("fhirVersions")) {
960      CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
961      if (npm.get("fhirVersions").isJsonArray()) {
962        for (JsonElement n : npm.getAsJsonArray("fhirVersions")) {
963          b.append(n.getAsString());
964        }
965      }
966      if (npm.get("fhirVersions").isJsonPrimitive()) {
967        b.append(npm.get("fhirVersions").getAsString());
968      }
969      return b.toString();
970    } else
971      return "";
972  }
973
974  public String dependencySummary() {
975    if (npm.has("dependencies")) {
976      CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
977      for (Entry<String, JsonElement> e : npm.getAsJsonObject("dependencies").entrySet()) {
978        b.append(e.getKey()+"#"+e.getValue().getAsString());
979      }
980      return b.toString();
981    } else
982      return "";
983  }
984
985  public void unPack(String dir) throws IOException {
986    unPack (dir, false);
987  }
988
989  public void unPackWithAppend(String dir) throws IOException {
990    unPack (dir, true);
991  }
992
993  public void unPack(String dir, boolean withAppend) throws IOException {
994    for (NpmPackageFolder folder : folders.values()) {
995      String dn = folder.getName();
996      if (!dn.equals("package") && (dn.startsWith("package/") || dn.startsWith("package\\"))) {
997        dn = dn.substring(8);
998      }
999      if (dn.equals("$root")) {
1000        dn = dir;
1001      } else {
1002         dn = Utilities.path(dir, dn);
1003      }
1004      Utilities.createDirectory(dn);
1005      for (String s : folder.listFiles()) {
1006        String fn = Utilities.path(dn, s);
1007        File f = new File(fn);
1008        if (withAppend && f.getName().startsWith("_append.")) {
1009          String appendFn = Utilities.path(dn, s.substring(8));
1010          if (new File(appendFn).exists())
1011            TextFile.appendBytesToFile(folder.fetchFile(s), appendFn);        
1012          else
1013            TextFile.bytesToFile(folder.fetchFile(s), appendFn);        
1014        } else
1015          TextFile.bytesToFile(folder.fetchFile(s), fn);
1016      }
1017//      if (path != null)
1018//        FileUtils.copyDirectory(new File(path), new File(dir));      
1019    }
1020  }
1021
1022  public void debugDump(String purpose) {
1023//    System.out.println("Debug Dump of Package for '"+purpose+"'. Path = "+path);
1024//    System.out.println("  npm = "+name()+"#"+version()+", canonical = "+canonical());
1025//    System.out.println("  folders = "+folders.size());
1026//    for (String s : sorted(folders.keySet())) {
1027//      NpmPackageFolder folder = folders.get(s);
1028//      System.out.println("    "+folder.dump());
1029//    }
1030  }
1031
1032  private List<String> sorted(Set<String> keys) {
1033    List<String> res = new ArrayList<String>();
1034    res.addAll(keys);
1035    Collections.sort(res);
1036    return res ;
1037  }
1038
1039  public void clearFolder(String folderName) {
1040    NpmPackageFolder folder = folders.get(folderName);
1041    folder.content.clear();
1042    folder.types.clear();    
1043  }
1044
1045  public void deleteFolder(String folderName) {
1046    folders.remove(folderName);
1047  }
1048
1049  public void addFile(String folderName, String name, byte[] cnt, String type) {
1050    if (!folders.containsKey(folderName)) {
1051      folders.put(folderName, new NpmPackageFolder(folderName));
1052    }
1053    NpmPackageFolder folder = folders.get(folderName);
1054    folder.content.put(name, cnt);
1055    if (!folder.types.containsKey(type))
1056      folder.types.put(type, new ArrayList<>());
1057    folder.types.get(type).add(name);
1058  }
1059
1060  public void loadAllFiles() throws IOException {
1061    for (String folder : folders.keySet()) {
1062      NpmPackageFolder pf = folders.get(folder);
1063      String p = folder.contains("$") ? path : Utilities.path(path, folder);
1064      for (File f : new File(p).listFiles()) {
1065        if (!f.isDirectory() && !isInternalExemptFile(f)) {
1066          pf.getContent().put(f.getName(), TextFile.fileToBytes(f));
1067        }
1068      }
1069    }
1070  }
1071
1072  public void loadAllFiles(ITransformingLoader loader) throws IOException {
1073    for (String folder : folders.keySet()) {
1074      NpmPackageFolder pf = folders.get(folder);
1075      String p = folder.contains("$") ? path : Utilities.path(path, folder);
1076      for (File f : new File(p).listFiles()) {
1077        if (!f.isDirectory() && !isInternalExemptFile(f)) {
1078          pf.getContent().put(f.getName(), loader.load(f));
1079        }
1080      }
1081    }
1082  }
1083
1084  public boolean isChangedByLoader() {
1085    return changedByLoader;
1086  }
1087
1088  public boolean isCore() {
1089    return "fhir.core".equals(JSONUtil.str(npm, "type"));
1090  }
1091
1092  public boolean hasCanonical(String url) {
1093    if (url == null) {
1094      return false;
1095    }
1096    String u = url.contains("|") ?  url.substring(0, url.indexOf("|")) : url;
1097    String v = url.contains("|") ?  url.substring(url.indexOf("|")+1) : null;
1098    NpmPackageFolder folder = folders.get("package");
1099    if (folder != null) {
1100      for (JsonElement e : folder.index.getAsJsonArray("files")) {
1101        JsonObject o = (JsonObject) e;
1102        if (u.equals(JSONUtil.str(o, "url"))) {
1103          if (v == null || v.equals(JSONUtil.str(o, "version"))) {
1104            return true;
1105          }
1106        }
1107      }
1108    }
1109    return false;
1110  }
1111
1112  public boolean canLazyLoad() throws IOException {
1113    for (NpmPackageFolder folder : folders.values()) {
1114      if (folder.folder == null) {        
1115        return false;
1116      }
1117    }
1118    if (Utilities.existsInList(name(), "fhir.test.data.r2", "fhir.test.data.r3", "fhir.test.data.r4", "fhir.tx.support.r2", "fhir.tx.support.r3", "fhir.tx.support.r4", "us.nlm.vsac")) {
1119      return true;
1120    }
1121    if (JSONUtil.bool(npm, "lazy-load")) {
1122      return true;
1123    }
1124    if (!hasFile("other", "spec.internals")) {
1125      return false;
1126    }
1127    return true;
1128  }
1129
1130  public boolean isNotForPublication() {
1131    return JSONUtil.bool(npm, "notForPublication");
1132 }
1133
1134  public InputStream load(PackageResourceInformation p) throws FileNotFoundException {
1135    return new FileInputStream(p.filename);
1136  }
1137  
1138  
1139}