001package org.hl7.fhir.utilities.npm;
002
003/*
004  Copyright (c) 2011+, HL7, Inc.
005  All rights reserved.
006  
007  Redistribution and use in source and binary forms, with or without modification, 
008  are permitted provided that the following conditions are met:
009    
010   * Redistributions of source code must retain the above copyright notice, this 
011     list of conditions and the following disclaimer.
012   * Redistributions in binary form must reproduce the above copyright notice, 
013     this list of conditions and the following disclaimer in the documentation 
014     and/or other materials provided with the distribution.
015   * Neither the name of HL7 nor the names of its contributors may be used to 
016     endorse or promote products derived from this software without specific 
017     prior written permission.
018  
019  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
020  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
021  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 
022  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
023  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 
024  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
025  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
026  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
027  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
028  POSSIBILITY OF SUCH DAMAGE.
029  
030 */
031
032
033import com.google.gson.GsonBuilder;
034import com.google.gson.JsonArray;
035import com.google.gson.JsonElement;
036import com.google.gson.JsonObject;
037import org.apache.commons.io.FileUtils;
038import org.hl7.fhir.exceptions.FHIRException;
039import org.hl7.fhir.utilities.SimpleHTTPClient;
040import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult;
041import org.hl7.fhir.utilities.IniFile;
042import org.hl7.fhir.utilities.TextFile;
043import org.hl7.fhir.utilities.Utilities;
044import org.hl7.fhir.utilities.VersionUtilities;
045import org.hl7.fhir.utilities.json.JSONUtil;
046import org.hl7.fhir.utilities.json.JsonTrackingParser;
047import org.hl7.fhir.utilities.npm.NpmPackage.NpmPackageFolder;
048import org.slf4j.Logger;
049import org.slf4j.LoggerFactory;
050
051import javax.net.ssl.*;
052
053import java.io.ByteArrayInputStream;
054import java.io.File;
055import java.io.FileInputStream;
056import java.io.FileNotFoundException;
057import java.io.IOException;
058import java.io.InputStream;
059import java.io.RandomAccessFile;
060import java.net.URL;
061import java.nio.channels.FileChannel;
062import java.nio.channels.FileLock;
063import java.sql.Timestamp;
064import java.text.ParseException;
065import java.text.SimpleDateFormat;
066import java.time.Instant;
067import java.util.*;
068import java.util.Map.Entry;
069
070/**
071 * This is a package cache manager implementation that uses a local disk cache
072 *
073 * <p>
074 * API:
075 * <p>
076 * constructor
077 * getPackageUrl
078 * getPackageId
079 * findPackageCache
080 * addPackageToCache
081 *
082 * @author Grahame Grieve
083 */
084public class FilesystemPackageCacheManager extends BasePackageCacheManager implements IPackageCacheManager {
085
086  //  private static final String SECONDARY_SERVER = "http://local.fhir.org:960/packages";
087  public static final String PACKAGE_REGEX = "^[a-zA-Z][A-Za-z0-9\\_\\-]*(\\.[A-Za-z0-9\\_\\-]+)+$";
088  public static final String PACKAGE_VERSION_REGEX = "^[A-Za-z][A-Za-z0-9\\_\\-]*(\\.[A-Za-z0-9\\_\\-]+)+\\#[A-Za-z0-9\\-\\_\\$]+(\\.[A-Za-z0-9\\-\\_\\$]+)*$";
089  public static final String PACKAGE_VERSION_REGEX_OPT = "^[A-Za-z][A-Za-z0-9\\_\\-]*(\\.[A-Za-z0-9\\_\\-]+)+(\\#[A-Za-z0-9\\-\\_]+(\\.[A-Za-z0-9\\-\\_]+)*)?$";
090  private static final Logger ourLog = LoggerFactory.getLogger(FilesystemPackageCacheManager.class);
091  private static final String CACHE_VERSION = "3"; // second version - see wiki page
092  private String cacheFolder;
093  private boolean progress = true;
094  private List<NpmPackage> temporaryPackages = new ArrayList<>();
095  private boolean buildLoaded = false;
096  private Map<String, String> ciList = new HashMap<String, String>();
097  private JsonArray buildInfo;
098
099  /**
100   * Constructor
101   */
102  public FilesystemPackageCacheManager(boolean userMode, int toolsVersion) throws IOException {
103    addPackageServer(PackageClient.PRIMARY_SERVER);
104    addPackageServer(PackageClient.SECONDARY_SERVER);
105
106    if (userMode)
107      cacheFolder = Utilities.path(System.getProperty("user.home"), ".fhir", "packages");
108    else
109      cacheFolder = Utilities.path("var", "lib", ".fhir", "packages");
110    if (!(new File(cacheFolder).exists()))
111      Utilities.createDirectory(cacheFolder);
112    if (!(new File(Utilities.path(cacheFolder, "packages.ini")).exists()))
113      TextFile.stringToFile("[cache]\r\nversion=" + CACHE_VERSION + "\r\n\r\n[urls]\r\n\r\n[local]\r\n\r\n", Utilities.path(cacheFolder, "packages.ini"), false);
114    createIniFile();
115  }
116
117  public void loadFromFolder(String packagesFolder) throws IOException {
118    File[] files = new File(packagesFolder).listFiles();
119    if (files != null) {
120      for (File f : files) {
121        if (f.getName().endsWith(".tgz")) {
122          temporaryPackages.add(NpmPackage.fromPackage(new FileInputStream(f)));
123        }
124      }
125    }
126  }
127
128  public String getFolder() {
129    return cacheFolder;
130  }
131
132  private List<String> sorted(String[] keys) {
133    List<String> names = new ArrayList<String>();
134    for (String s : keys)
135      names.add(s);
136    Collections.sort(names);
137    return names;
138  }
139
140  private List<String> reverseSorted(String[] keys) {
141    Arrays.sort(keys, Collections.reverseOrder());
142    return Arrays.asList(keys);
143  }
144
145  private NpmPackage loadPackageInfo(String path) throws IOException {
146    NpmPackage pi = NpmPackage.fromFolder(path);
147    return pi;
148  }
149
150  private void clearCache() throws IOException {
151    for (File f : new File(cacheFolder).listFiles()) {
152      if (f.isDirectory()) {
153        new CacheLock(f.getName()).doWithLock(() -> {
154          Utilities.clearDirectory(f.getAbsolutePath());
155          try {
156            FileUtils.deleteDirectory(f);
157          } catch (Exception e1) {
158            try {
159              FileUtils.deleteDirectory(f);
160            } catch (Exception e2) {
161              // just give up
162            }
163          }
164          return null; // must return something
165        });
166      } else if (!f.getName().equals("packages.ini"))
167        FileUtils.forceDelete(f);
168    }
169    IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
170    ini.removeSection("packages");
171    ini.save();
172  }
173
174  private void createIniFile() throws IOException {
175    IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
176    boolean save = false;
177    String v = ini.getStringProperty("cache", "version");
178    if (!CACHE_VERSION.equals(v)) {
179      clearCache();
180      ini.setStringProperty("cache", "version", CACHE_VERSION, null);
181      ini.save();
182    }
183  }
184
185  private void checkValidVersionString(String version, String id) {
186    if (Utilities.noString(version)) {
187      throw new FHIRException("Cannot add package " + id + " to the package cache - a version must be provided");
188    }
189    if (version.startsWith("file:")) {
190      throw new FHIRException("Cannot add package " + id + " to the package cache - the version '" + version + "' is illegal in this context");
191    }
192    for (char ch : version.toCharArray()) {
193      if (!Character.isAlphabetic(ch) && !Character.isDigit(ch) && !Utilities.existsInList(ch, '.', '-', '$')) {
194        throw new FHIRException("Cannot add package " + id + " to the package cache - the version '" + version + "' is illegal (ch '" + ch + "'");
195      }
196    }
197  }
198
199  private void listSpecs(Map<String, String> specList, String server) throws IOException {
200    CachingPackageClient pc = new CachingPackageClient(server);
201    List<PackageInfo> matches = pc.search(null, null, null, false);
202    for (PackageInfo m : matches) {
203      if (!specList.containsKey(m.getId())) {
204        specList.put(m.getId(), m.getUrl());
205      }
206    }
207  }
208
209  protected InputStreamWithSrc loadFromPackageServer(String id, String version) {
210    InputStreamWithSrc retVal = super.loadFromPackageServer(id, version);
211    if (retVal != null) {
212      return retVal;
213    }
214
215    retVal = super.loadFromPackageServer(id, VersionUtilities.getMajMin(version)+".x");
216    if (retVal != null) {
217      return retVal;
218    }
219
220    // ok, well, we'll try the old way
221    return fetchTheOldWay(id, version);
222  }
223
224  public String getLatestVersion(String id) throws IOException {
225    for (String nextPackageServer : getPackageServers()) {
226      // special case:
227      if (!(CommonPackages.ID_PUBPACK.equals(id) && PackageClient.PRIMARY_SERVER.equals(nextPackageServer))) {
228        CachingPackageClient pc = new CachingPackageClient(nextPackageServer);
229        try {
230          return pc.getLatestVersion(id);
231        } catch (IOException e) {
232          ourLog.info("Failed to determine latest version of package {} from server: {}", id, nextPackageServer);
233        }
234      }
235    }
236
237    return fetchVersionTheOldWay(id);
238  }
239
240  private NpmPackage loadPackageFromFile(String id, String folder) throws IOException {
241    File f = new File(Utilities.path(folder, id));
242    if (!f.exists()) {
243      throw new FHIRException("Package '" + id + "  not found in folder " + folder);
244    }
245    if (!f.isDirectory()) {
246      throw new FHIRException("File for '" + id + "  found in folder " + folder + ", not a folder");
247    }
248    File fp = new File(Utilities.path(folder, id, "package", "package.json"));
249    if (!fp.exists()) {
250      throw new FHIRException("Package '" + id + "  found in folder " + folder + ", but does not contain a package.json file in /package");
251    }
252    return NpmPackage.fromFolder(f.getAbsolutePath());
253  }
254
255  /**
256   * Clear the cache
257   *
258   * @throws IOException
259   */
260  public void clear() throws IOException {
261    clearCache();
262  }
263
264  // ========================= Utilities ============================================================================
265
266  /**
267   * Remove a particular package from the cache
268   *
269   * @param id
270   * @param ver
271   * @throws IOException
272   */
273  public void removePackage(String id, String ver) throws IOException {
274    new CacheLock(id + "#" + ver).doWithLock(() -> {
275      String f = Utilities.path(cacheFolder, id + "#" + ver);
276      File ff = new File(f);
277      if (ff.exists()) {
278        Utilities.clearDirectory(f);
279        IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
280        ini.removeProperty("packages", id + "#" + ver);
281        ini.save();
282        ff.delete();
283      }
284      return null;
285    });
286  }
287
288  /**
289   * Load the identified package from the cache - if it exists
290   * <p>
291   * This is for special purpose only (testing, control over speed of loading).
292   * Generally, use the loadPackage method
293   *
294   * @param id
295   * @param version
296   * @return
297   * @throws IOException
298   */
299  @Override
300  public NpmPackage loadPackageFromCacheOnly(String id, String version) throws IOException {
301    if (!Utilities.noString(version) && version.startsWith("file:")) {
302      return loadPackageFromFile(id, version.substring(5));
303    }
304
305    for (NpmPackage p : temporaryPackages) {
306      if (p.name().equals(id) && ("current".equals(version) || "dev".equals(version) || p.version().equals(version))) {
307        return p;
308      }
309      if (p.name().equals(id) && Utilities.noString(version)) {
310        return p;
311      }
312    }
313    String foundPackage = null;
314    String foundVersion = null;
315    for (String f : reverseSorted(new File(cacheFolder).list())) {
316      File cf = new File(Utilities.path(cacheFolder, f));
317      if (cf.isDirectory()) {
318        if (f.equals(id + "#" + version) || (Utilities.noString(version) && f.startsWith(id + "#"))) {
319          return loadPackageInfo(Utilities.path(cacheFolder, f));
320        }
321        if (version != null && version.endsWith(".x") && f.contains("#")) {
322          String[] parts = f.split("#");
323          if (parts[0].equals(id) && VersionUtilities.isMajMinOrLaterPatch((foundVersion!=null ? foundVersion : version),parts[1])) {
324            foundVersion = parts[1];
325            foundPackage = f;
326          }
327        }
328      }
329    }
330    if (foundPackage!=null) {
331      return loadPackageInfo(Utilities.path(cacheFolder, foundPackage));
332    }
333    if ("dev".equals(version))
334      return loadPackageFromCacheOnly(id, "current");
335    else
336      return null;
337  }
338
339  /**
340   * Add an already fetched package to the cache
341   */
342  @Override
343  public NpmPackage addPackageToCache(String id, String version, InputStream packageTgzInputStream, String sourceDesc) throws IOException {
344    checkValidVersionString(version, id);
345    if (progress) {
346      System.out.println("Installing " + id + "#" + (version == null ? "?" : version) + " to the package cache");
347      System.out.print("  Fetching:");
348    }
349
350    NpmPackage npm = NpmPackage.fromPackage(packageTgzInputStream, sourceDesc, true);
351
352    if (progress) {
353      System.out.println();
354      System.out.print("  Installing: ");
355    }
356    
357    if (npm.name() == null || id == null || !id.equalsIgnoreCase(npm.name())) {
358      if (!id.equals("hl7.fhir.r5.core") && !id.equals("hl7.fhir.us.immds")) {// temporary work around
359        throw new IOException("Attempt to import a mis-identified package. Expected " + id + ", got " + npm.name());
360      }
361    }
362    if (version == null)
363      version = npm.version();
364
365    String v = version;
366    return new CacheLock(id + "#" + version).doWithLock(() -> {
367      NpmPackage pck = null;
368      String packRoot = Utilities.path(cacheFolder, id + "#" + v);
369      try {
370        // ok, now we have a lock on it... check if something created it while we were waiting
371        if (!new File(packRoot).exists() || Utilities.existsInList(v, "current", "dev")) {
372          Utilities.createDirectory(packRoot);
373          try {
374            Utilities.clearDirectory(packRoot);
375          } catch (Throwable t) {
376            System.out.println("Unable to clear directory: "+packRoot+": "+t.getMessage()+" - this may cause problems later");
377          }
378
379          int i = 0;
380          int c = 0;
381          int size = 0;
382          for (Entry<String, NpmPackageFolder> e : npm.getFolders().entrySet()) {
383            String dir = e.getKey().equals("package") ? Utilities.path(packRoot, "package") : Utilities.path(packRoot, "package", e.getKey());
384            if (!(new File(dir).exists()))
385              Utilities.createDirectory(dir);
386            for (Entry<String, byte[]> fe : e.getValue().getContent().entrySet()) {
387              String fn = Utilities.path(dir, Utilities.cleanFileName(fe.getKey()));
388              byte[] cnt = fe.getValue();
389              TextFile.bytesToFile(cnt, fn);
390              size = size + cnt.length;
391              i++;
392              if (progress && i % 50 == 0) {
393                c++;
394                System.out.print(".");
395                if (c == 120) {
396                  System.out.println("");
397                  System.out.print("  ");
398                  c = 2;
399                }
400              }
401            }
402          }
403
404
405          IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
406          ini.setTimeStampFormat("yyyyMMddhhmmss");
407          ini.setTimestampProperty("packages", id + "#" + v, Timestamp.from(Instant.now()), null);
408          ini.setIntegerProperty("package-sizes", id + "#" + v, size, null);
409          ini.save();
410          if (progress)
411            System.out.println(" done.");
412        }
413        pck = loadPackageInfo(packRoot);
414        if (!id.equals(JSONUtil.str(npm.getNpm(), "name")) || !v.equals(JSONUtil.str(npm.getNpm(), "version"))) {
415          if (!id.equals(JSONUtil.str(npm.getNpm(), "name"))) {
416            npm.getNpm().addProperty("original-name", JSONUtil.str(npm.getNpm(), "name"));
417            npm.getNpm().remove("name");
418            npm.getNpm().addProperty("name", id);
419          }
420          if (!v.equals(JSONUtil.str(npm.getNpm(), "version"))) {
421            npm.getNpm().addProperty("original-version", JSONUtil.str(npm.getNpm(), "version"));
422            npm.getNpm().remove("version");
423            npm.getNpm().addProperty("version", v);
424          }
425          TextFile.stringToFile(new GsonBuilder().setPrettyPrinting().create().toJson(npm.getNpm()), Utilities.path(cacheFolder, id + "#" + v, "package", "package.json"), false);
426        }
427      } catch (Exception e) {
428        try {
429          // don't leave a half extracted package behind
430          System.out.println("Clean up package " + packRoot + " because installation failed: " + e.getMessage());
431          e.printStackTrace();
432          Utilities.clearDirectory(packRoot);
433          new File(packRoot).delete();
434        } catch (Exception ei) {
435          // nothing
436        }
437        throw e;
438      }
439      return pck;
440    });
441  }
442
443  @Override
444  public String getPackageUrl(String packageId) throws IOException {
445    String result = super.getPackageUrl(packageId);
446    if (result == null) {
447      result = getPackageUrlFromBuildList(packageId);
448    }
449
450    return result;
451  }
452
453  public void listAllIds(Map<String, String> specList) throws IOException {
454    for (NpmPackage p : temporaryPackages) {
455      specList.put(p.name(), p.canonical());
456    }
457    for (String next : getPackageServers()) {
458      listSpecs(specList, next);
459    }
460    addCIBuildSpecs(specList);
461  }
462
463  @Override
464  public NpmPackage loadPackage(String id, String version) throws FHIRException, IOException {
465    //ok, try to resolve locally
466    if (!Utilities.noString(version) && version.startsWith("file:")) {
467      return loadPackageFromFile(id, version.substring(5));
468    }
469
470    if (version == null && id.contains("#")) {
471      version = id.substring(id.indexOf("#")+1);
472      id = id.substring(0, id.indexOf("#"));
473    }
474
475    if (version == null) {
476      try {
477        version = getLatestVersion(id);
478      } catch (Exception e) {
479        version = null;
480      }
481    }
482    NpmPackage p = loadPackageFromCacheOnly(id, version);
483    if (p != null) {
484      if ("current".equals(version)) {
485        p = checkCurrency(id, p);
486      }
487      if (p != null)
488        return p;
489    }
490
491    if ("dev".equals(version)) {
492      p = loadPackageFromCacheOnly(id, "current");
493      p = checkCurrency(id, p);
494      if (p != null)
495        return p;
496      version = "current";
497    }
498
499    // nup, don't have it locally (or it's expired)
500    FilesystemPackageCacheManager.InputStreamWithSrc source;
501    if ("current".equals(version) || version.startsWith("current$")) {
502      // special case - fetch from ci-build server
503      source = loadFromCIBuild(id, version.startsWith("current$") ? version.substring(8) : null);
504    } else {
505      source = loadFromPackageServer(id, version);
506    }
507    if (source == null) {
508      throw new FHIRException("Unable to find package "+id+"#"+version);
509    }
510    return addPackageToCache(id, version == null ? source.version : version, source.stream, source.url);
511  }
512
513  private InputStream fetchFromUrlSpecific(String source, boolean optional) throws FHIRException {
514    try {
515      SimpleHTTPClient http = new SimpleHTTPClient();
516      HTTPResult res = http.get(source);
517      res.checkThrowException();
518      return new ByteArrayInputStream(res.getContent());
519    } catch (Exception e) {
520      if (optional)
521        return null;
522      else
523        throw new FHIRException("Unable to fetch: "+e.getMessage(), e);
524    }
525  }
526
527  private InputStreamWithSrc loadFromCIBuild(String id, String branch) throws IOException {
528    checkBuildLoaded();
529    if (ciList.containsKey(id)) {
530      if (branch == null) {
531        InputStream stream = fetchFromUrlSpecific(Utilities.pathURL(ciList.get(id), "package.tgz"), false);
532        return new InputStreamWithSrc(stream, Utilities.pathURL(ciList.get(id), "package.tgz"), "current");
533      } else {
534        InputStream stream = fetchFromUrlSpecific(Utilities.pathURL(ciList.get(id), "branches", branch, "package.tgz"), false);
535        return new InputStreamWithSrc(stream, Utilities.pathURL(ciList.get(id), "branches", branch, "package.tgz"), "current$"+branch);
536      }
537    } else if (id.startsWith("hl7.fhir.r5")) {
538      InputStream stream = fetchFromUrlSpecific(Utilities.pathURL("http://build.fhir.org", id + ".tgz"), false);
539      return new InputStreamWithSrc(stream, Utilities.pathURL("http://build.fhir.org", id + ".tgz"), "current");
540    } else {
541      throw new FHIRException("The package '" + id + "' has no entry on the current build server ("+ciList.toString()+")");
542    }
543  }
544
545  private String getPackageUrlFromBuildList(String packageId) throws IOException {
546    checkBuildLoaded();
547    for (JsonElement n : buildInfo) {
548      JsonObject o = (JsonObject) n;
549      if (packageId.equals(JSONUtil.str(o, "package-id"))) {
550        return JSONUtil.str(o, "url");
551      }
552    }
553    return null;
554  }
555
556  private void addCIBuildSpecs(Map<String, String> specList) throws IOException {
557    checkBuildLoaded();
558    for (JsonElement n : buildInfo) {
559      JsonObject o = (JsonObject) n;
560      if (!specList.containsKey(JSONUtil.str(o, "package-id"))) {
561        specList.put(JSONUtil.str(o, "package-id"), JSONUtil.str(o, "url"));
562      }
563    }
564  }
565
566  @Override
567  public String getPackageId(String canonicalUrl) throws IOException {
568    String retVal = findCanonicalInLocalCache(canonicalUrl);
569    
570    if(retVal == null) {
571      retVal = super.getPackageId(canonicalUrl);
572    }
573    
574    if (retVal == null) {
575      retVal = getPackageIdFromBuildList(canonicalUrl);
576    }
577
578    return retVal;
579  }
580
581
582  public String findCanonicalInLocalCache(String canonicalUrl) {
583    try {
584      for (String pf : listPackages()) {
585        if (new File(Utilities.path(cacheFolder, pf, "package", "package.json")).exists()) {
586          JsonObject npm = JsonTrackingParser.parseJsonFile(Utilities.path(cacheFolder, pf, "package", "package.json"));
587          if (canonicalUrl.equals(JSONUtil.str(npm, "canonical"))) {
588            return JSONUtil.str(npm, "name");
589          }
590        }
591      }
592    } catch (IOException e) {
593    }
594    return null;
595  }
596
597  // ========================= Package Mgmt API =======================================================================
598
599  private String getPackageIdFromBuildList(String canonical) throws IOException {
600    if (canonical == null) {
601      return null;
602    }
603    checkBuildLoaded();
604    if (buildInfo != null) {
605      for (JsonElement n : buildInfo) {
606        JsonObject o = (JsonObject) n;
607        if (canonical.equals(JSONUtil.str(o, "url"))) {
608          return JSONUtil.str(o, "package-id");
609        }
610      }
611      for (JsonElement n : buildInfo) {
612        JsonObject o = (JsonObject) n;
613        if (JSONUtil.str(o, "url").startsWith(canonical + "/ImplementationGuide/")) {
614          return JSONUtil.str(o, "package-id");
615        }
616      }
617    }
618    return null;
619  }
620
621  private NpmPackage checkCurrency(String id, NpmPackage p) throws IOException {
622    checkBuildLoaded();
623    // special case: current versions roll over, and we have to check their currency
624    try {
625      String url = ciList.get(id);
626      JsonObject json = JsonTrackingParser.fetchJson(Utilities.pathURL(url, "package.manifest.json"));
627      String currDate = JSONUtil.str(json, "date");
628      String packDate = p.date();
629      if (!currDate.equals(packDate))
630        return null; // nup, we need a new copy
631      return p;
632    } catch (Exception e) {
633      return p;
634    }
635  }
636
637  private boolean checkBuildLoaded() {
638    if (buildLoaded)
639      return true;
640    try {
641      loadFromBuildServer();
642    } catch (Exception e) {
643      System.out.println("Error connecting to build server - running without build (" + e.getMessage() + ")");
644      e.printStackTrace();
645    }
646    return false;
647  }
648
649  private void loadFromBuildServer() throws IOException {
650    SimpleHTTPClient http = new SimpleHTTPClient();
651    http.trustAllhosts();
652    HTTPResult res = http.get("https://build.fhir.org/ig/qas.json?nocache=" + System.currentTimeMillis());
653    res.checkThrowException();
654
655    buildInfo = (JsonArray) new com.google.gson.JsonParser().parse(TextFile.bytesToString(res.getContent()));
656
657    List<BuildRecord> builds = new ArrayList<>();
658
659    for (JsonElement n : buildInfo) {
660      JsonObject o = (JsonObject) n;
661      if (o.has("url") && o.has("package-id") && o.get("package-id").getAsString().contains(".")) {
662        String u = o.get("url").getAsString();
663        if (u.contains("/ImplementationGuide/"))
664          u = u.substring(0, u.indexOf("/ImplementationGuide/"));
665        builds.add(new BuildRecord(u, o.get("package-id").getAsString(), getRepo(o.get("repo").getAsString()), readDate(o.get("date").getAsString())));
666      }
667    }
668    Collections.sort(builds, new BuildRecordSorter());
669    for (BuildRecord bld : builds) {
670      if (!ciList.containsKey(bld.getPackageId())) {
671        ciList.put(bld.getPackageId(), "https://build.fhir.org/ig/" + bld.getRepo());
672      }
673    }
674    buildLoaded = true; // whether it succeeds or not
675  }
676
677  private String getRepo(String path) {
678    String[] p = path.split("\\/");
679    return p[0] + "/" + p[1];
680  }
681
682  private Date readDate(String s) {
683    SimpleDateFormat sdf = new SimpleDateFormat("EEE, dd MMM, yyyy HH:mm:ss Z", new Locale("en", "US"));
684    try {
685      return sdf.parse(s);
686    } catch (ParseException e) {
687      e.printStackTrace();
688      return new Date();
689    }
690  }
691
692  // ----- the old way, from before package server, while everything gets onto the package server
693  private InputStreamWithSrc fetchTheOldWay(String id, String v) {
694    String url = getUrlForPackage(id);
695    if (url == null) {
696      try {
697        url = getPackageUrlFromBuildList(id);
698      } catch (Exception e) {
699        url = null;
700      }
701    }
702    if (url == null) {
703      throw new FHIRException("Unable to resolve package id " + id + "#" + v);
704    }
705    if (url.contains("/ImplementationGuide/")) {
706      url = url.substring(0, url.indexOf("/ImplementationGuide/"));
707    }
708    String pu = Utilities.pathURL(url, "package-list.json");
709    String aurl = pu;
710    JsonObject json;
711    try {
712      json = JsonTrackingParser.fetchJson(pu);
713    } catch (Exception e) {
714      String pv = Utilities.pathURL(url, v, "package.tgz");
715      try {
716        aurl = pv;
717        InputStreamWithSrc src = new InputStreamWithSrc(fetchFromUrlSpecific(pv, false), pv, v);
718        return src;
719      } catch (Exception e1) {
720        throw new FHIRException("Error fetching package directly (" + pv + "), or fetching package list for " + id + " from " + pu + ": " + e1.getMessage(), e1);
721      }
722    }
723    if (!id.equals(JSONUtil.str(json, "package-id")))
724      throw new FHIRException("Package ids do not match in " + pu + ": " + id + " vs " + JSONUtil.str(json, "package-id"));
725    for (JsonElement e : json.getAsJsonArray("list")) {
726      JsonObject vo = (JsonObject) e;
727      if (v.equals(JSONUtil.str(vo, "version"))) {
728        aurl = Utilities.pathURL(JSONUtil.str(vo, "path"), "package.tgz");
729        String u = Utilities.pathURL(JSONUtil.str(vo, "path"), "package.tgz");
730        return new InputStreamWithSrc(fetchFromUrlSpecific(u, true), u, v);
731      }
732    }
733
734    return null;
735  }
736
737
738  // ---------- Current Build SubSystem --------------------------------------------------------------------------------------
739
740  private String fetchVersionTheOldWay(String id) throws IOException {
741    String url = getUrlForPackage(id);
742    if (url == null) {
743      try {
744        url = getPackageUrlFromBuildList(id);
745      } catch (Exception e) {
746        url = null;
747      }
748    }
749    if (url == null) {
750      throw new FHIRException("Unable to resolve package id " + id);
751    }
752    String pu = Utilities.pathURL(url, "package-list.json");
753    JsonObject json = JsonTrackingParser.fetchJson(pu);
754    if (!id.equals(JSONUtil.str(json, "package-id")))
755      throw new FHIRException("Package ids do not match in " + pu + ": " + id + " vs " + JSONUtil.str(json, "package-id"));
756    for (JsonElement e : json.getAsJsonArray("list")) {
757      JsonObject vo = (JsonObject) e;
758      if (JSONUtil.bool(vo, "current")) {
759        return JSONUtil.str(vo, "version");
760      }
761    }
762
763    return null;
764  }
765
766  private String getUrlForPackage(String id) {
767    if (CommonPackages.ID_XVER.equals(id)) {
768      return "http://fhir.org/packages/hl7.fhir.xver-extensions";
769    }
770    return null;
771  }
772
773  public List<String> listPackages() {
774    List<String> res = new ArrayList<>();
775    for (File f : new File(cacheFolder).listFiles()) {
776      if (f.isDirectory() && f.getName().contains("#")) {
777        res.add(f.getName());
778      }
779    }
780    return res;
781  }
782
783  /**
784   * if you don't provide and implementation of this interface, the PackageCacheManager will use the web directly.
785   * <p>
786   * You can use this interface to
787   *
788   * @author graha
789   */
790  public interface INetworkServices {
791
792    InputStream resolvePackage(String packageId, String version);
793  }
794
795  public interface CacheLockFunction<T> {
796    T get() throws IOException;
797  }
798
799  public class BuildRecordSorter implements Comparator<BuildRecord> {
800
801    @Override
802    public int compare(BuildRecord arg0, BuildRecord arg1) {
803      return arg1.date.compareTo(arg0.date);
804    }
805  }
806
807  public class BuildRecord {
808
809    private String url;
810    private String packageId;
811    private String repo;
812    private Date date;
813
814    public BuildRecord(String url, String packageId, String repo, Date date) {
815      super();
816      this.url = url;
817      this.packageId = packageId;
818      this.repo = repo;
819      this.date = date;
820    }
821
822    public String getUrl() {
823      return url;
824    }
825
826    public String getPackageId() {
827      return packageId;
828    }
829
830    public String getRepo() {
831      return repo;
832    }
833
834    public Date getDate() {
835      return date;
836    }
837
838
839  }
840
841
842
843  public class VersionHistory {
844    private String id;
845    private String canonical;
846    private String current;
847    private Map<String, String> versions = new HashMap<>();
848
849    public String getCanonical() {
850      return canonical;
851    }
852
853    public String getCurrent() {
854      return current;
855    }
856
857    public Map<String, String> getVersions() {
858      return versions;
859    }
860
861    public String getId() {
862      return id;
863    }
864  }
865
866  public class PackageEntry {
867
868    private byte[] bytes;
869    private String name;
870
871    public PackageEntry(String name) {
872      this.name = name;
873    }
874
875    public PackageEntry(String name, byte[] bytes) {
876      this.name = name;
877      this.bytes = bytes;
878    }
879  }
880
881  public class CacheLock {
882
883    private final File lockFile;
884
885    public CacheLock(String name) throws IOException {
886      this.lockFile = new File(cacheFolder, name + ".lock");
887      if (!lockFile.isFile()) {
888        TextFile.stringToFile("", lockFile);
889      }
890    }
891
892    public <T> T doWithLock(CacheLockFunction<T> f) throws FileNotFoundException, IOException {
893      try (FileChannel channel = new RandomAccessFile(lockFile, "rw").getChannel()) {
894        final FileLock fileLock = channel.lock();
895        T result = null;
896        try {
897          result = f.get();
898        } finally {
899          fileLock.release();
900        }
901        if (!lockFile.delete()) {
902          lockFile.deleteOnExit();
903        }
904        return result;
905      }
906    }
907  }
908
909  public boolean packageExists(String id, String ver) throws IOException {
910    if (packageInstalled(id, ver)) {
911      return true;
912    }
913    for (String s : getPackageServers()) {
914      if (new PackageClient(s).exists(id, ver)) {
915        return true;
916      }
917    }
918    return false;
919  }
920
921  public boolean packageInstalled(String id, String version) {
922    for (NpmPackage p : temporaryPackages) {
923      if (p.name().equals(id) && ("current".equals(version) || "dev".equals(version) || p.version().equals(version))) {
924        return true;
925      }
926      if (p.name().equals(id) && Utilities.noString(version)) {
927        return true;
928      }
929    }
930
931    for (String f : sorted(new File(cacheFolder).list())) {
932      if (f.equals(id + "#" + version) || (Utilities.noString(version) && f.startsWith(id + "#"))) {
933        return true;
934      }
935    }
936    if ("dev".equals(version))
937      return packageInstalled(id, "current");
938    else
939      return false;
940  }
941
942}