001package org.hl7.fhir.convertors.analytics;
002
003import java.io.IOException;
004import java.util.ArrayList;
005import java.util.HashSet;
006import java.util.List;
007import java.util.Set;
008
009import javax.xml.parsers.ParserConfigurationException;
010
011import org.hl7.fhir.exceptions.FHIRException;
012import org.hl7.fhir.utilities.SimpleHTTPClient;
013import org.hl7.fhir.utilities.TextFile;
014import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult;
015import org.hl7.fhir.utilities.json.JSONUtil;
016import org.hl7.fhir.utilities.json.JsonTrackingParser;
017import org.hl7.fhir.utilities.npm.FilesystemPackageCacheManager;
018import org.hl7.fhir.utilities.npm.NpmPackage;
019import org.hl7.fhir.utilities.npm.PackageClient;
020import org.hl7.fhir.utilities.npm.PackageInfo;
021import org.hl7.fhir.utilities.npm.ToolsVersion;
022import org.hl7.fhir.utilities.xml.XMLUtil;
023import org.w3c.dom.Document;
024import org.w3c.dom.Element;
025import org.xml.sax.SAXException;
026
027import com.google.gson.JsonArray;
028import com.google.gson.JsonObject;
029
030public class PackageVisitor {
031  
032  public interface IPackageVisitorProcessor {
033     public void processResource(String pid, String version, String type, byte[] content) throws FHIRException;
034  }
035
036  private List<String> resourceTypes = new ArrayList<>();
037  private List<String> versions = new ArrayList<>();
038  private boolean corePackages;
039  private boolean oldVersions;
040  private IPackageVisitorProcessor processor;
041  private FilesystemPackageCacheManager pcm;
042  private PackageClient pc;  
043  
044  public List<String> getResourceTypes() {
045    return resourceTypes;
046  }
047
048  public void setResourceTypes(List<String> resourceTypes) {
049    this.resourceTypes = resourceTypes;
050  }
051
052  public List<String> getVersions() {
053    return versions;
054  }
055
056  public void setVersions(List<String> versions) {
057    this.versions = versions;
058  }
059
060
061
062
063  public boolean isCorePackages() {
064    return corePackages;
065  }
066
067
068
069
070  public void setCorePackages(boolean corePackages) {
071    this.corePackages = corePackages;
072  }
073
074
075
076
077  public boolean isOldVersions() {
078    return oldVersions;
079  }
080
081
082
083
084  public void setOldVersions(boolean oldVersions) {
085    this.oldVersions = oldVersions;
086  }
087
088
089
090
091  public IPackageVisitorProcessor getProcessor() {
092    return processor;
093  }
094
095  public void setProcessor(IPackageVisitorProcessor processor) {
096    this.processor = processor;
097  }
098
099  public void visitPackages() throws IOException, ParserConfigurationException, SAXException {
100    System.out.println("Finding packages");
101    pc = new PackageClient(PackageClient.PRIMARY_SERVER);
102    pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
103    Set<String> pidList = getAllPackages();
104    System.out.println("Go: "+pidList.size()+" packages");
105    for (String pid : pidList) {
106      List<String> vList = listVersions(pid);
107      if (oldVersions) {
108        for (String v : vList) {
109          processPackage(pid, v);          
110        }
111      } else if (vList.isEmpty()) {
112        System.out.println("No Packages for "+pid);
113      } else {
114        processPackage(pid, vList.get(vList.size() - 1));
115      }
116    }         
117  }
118
119  private List<String> listVersions(String pid) throws IOException {
120    List<String> list = new ArrayList<>();
121    if (pid !=null) {
122      for (PackageInfo i : pc.getVersions(pid)) {
123        list.add(i.getVersion());
124      }    
125    }
126    return list;
127  }
128
129  private Set<String> getAllPackages() throws IOException, ParserConfigurationException, SAXException {
130    Set<String> list = new HashSet<>();
131    for (PackageInfo i : pc.search(null, null, null, false)) {
132      list.add(i.getId());
133    }    
134    JsonObject json = JsonTrackingParser.fetchJson("https://raw.githubusercontent.com/FHIR/ig-registry/master/fhir-ig-list.json");
135    for (JsonObject ig : JSONUtil.objects(json, "guides")) {
136      list.add(JSONUtil.str(ig, "npm-name"));
137    }
138    json = JsonTrackingParser.fetchJson("https://raw.githubusercontent.com/FHIR/ig-registry/master/package-feeds.json");
139    for (JsonObject feed : JSONUtil.objects(json, "feeds")) {
140      processFeed(list, JSONUtil.str(feed, "url"));
141    }
142    
143    return list;
144  }
145
146  private void processFeed(Set<String> list, String str) throws IOException, ParserConfigurationException, SAXException {
147    System.out.println("Feed "+str);
148    try {
149      SimpleHTTPClient fetcher = new SimpleHTTPClient();
150      HTTPResult res = fetcher.get(str+"?nocache=" + System.currentTimeMillis());
151      res.checkThrowException();
152      Document xml = XMLUtil.parseToDom(res.getContent());
153      for (Element channel : XMLUtil.getNamedChildren(xml.getDocumentElement(), "channel")) {
154        for (Element item : XMLUtil.getNamedChildren(channel, "item")) {
155          String pid = XMLUtil.getNamedChildText(item, "title");
156          if (pid.contains("#")) {
157            list.add(pid.substring(0, pid.indexOf("#")));
158          }
159        }
160      }
161    } catch (Exception e) {
162      System.out.println("   "+e.getMessage());
163    }
164  }
165
166
167  private void processPackage(String pid, String v) throws IOException {
168    NpmPackage npm = null;
169    String fv = null;
170    try {
171      npm = pcm.loadPackage(pid, v);
172      fv = npm.fhirVersion();
173    } catch (Throwable e) {
174      System.out.println("Unable to process: "+pid+"#"+v+": "+e.getMessage());      
175    }
176    int c = 0;
177    if (fv != null && (versions.isEmpty() || versions.contains(fv))) {
178      for (String type : resourceTypes) {
179        for (String s : npm.listResources(type)) {
180          c++;
181          processor.processResource(pid+"#"+v, fv, type, TextFile.streamToBytes(npm.load("package", s)));
182        }
183      }
184    }    
185    System.out.println("Processed: "+pid+"#"+v+": "+c+" resources");      
186  }
187
188}