001package org.hl7.fhir.r4.context;
002
003import java.io.File;
004import java.io.FileNotFoundException;
005import java.io.FileWriter;
006import java.io.IOException;
007import java.text.DateFormat;
008import java.text.ParseException;
009import java.text.SimpleDateFormat;
010import java.util.ArrayList;
011import java.util.Date;
012import java.util.HashMap;
013import java.util.List;
014import java.util.Map;
015
016import org.apache.commons.codec.Charsets;
017import org.apache.commons.lang3.StringUtils;
018import org.hl7.fhir.exceptions.FHIRException;
019import org.hl7.fhir.exceptions.FHIRFormatError;
020import org.hl7.fhir.r4.context.IWorkerContext.ValidationResult;
021import org.hl7.fhir.r4.formats.IParser.OutputStyle;
022import org.hl7.fhir.r4.formats.JsonParser;
023import org.hl7.fhir.r4.model.CodeSystem.ConceptDefinitionComponent;
024import org.hl7.fhir.r4.model.CodeableConcept;
025import org.hl7.fhir.r4.model.Coding;
026import org.hl7.fhir.r4.model.UriType;
027import org.hl7.fhir.r4.model.ValueSet;
028import org.hl7.fhir.r4.model.ValueSet.ConceptSetComponent;
029import org.hl7.fhir.r4.model.ValueSet.ConceptSetFilterComponent;
030import org.hl7.fhir.r4.model.ValueSet.ValueSetExpansionContainsComponent;
031import org.hl7.fhir.r4.terminologies.ValueSetExpander.TerminologyServiceErrorClass;
032import org.hl7.fhir.r4.terminologies.ValueSetExpander.ValueSetExpansionOutcome;
033import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
034import org.hl7.fhir.utilities.TextFile;
035import org.hl7.fhir.utilities.Utilities;
036import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity;
037
038import com.google.gson.JsonElement;
039import com.google.gson.JsonNull;
040import com.google.gson.JsonObject;
041import com.google.gson.JsonPrimitive;
042
043/**
044 * This implements a two level cache. 
045 *  - a temporary cache for remmbering previous local operations
046 *  - a persistent cache for rembering tx server operations
047 *  
048 * the cache is a series of pairs: a map, and a list. the map is the loaded cache, the list is the persiistent cache, carefully maintained in order for version control consistency
049 * 
050 * @author graha
051 *
052 */
053public class TerminologyCache {
054  public static final boolean TRANSIENT = false;
055  public static final boolean PERMANENT = true;
056  private static final String NAME_FOR_NO_SYSTEM = "all-systems";
057  private static final String ENTRY_MARKER = "-------------------------------------------------------------------------------------";
058  private static final String BREAK = "####";
059
060  public class CacheToken {
061    private String name;
062    private String key;
063    private String request;
064    public void setName(String n) {
065      if (name == null)
066        name = n;
067      else if (!n.equals(name))
068        name = NAME_FOR_NO_SYSTEM;
069    }
070  }
071
072  private class CacheEntry {
073    private String request;
074    private boolean persistent;
075    private ValidationResult v;
076    private ValueSetExpansionOutcome e;
077  }
078  
079  private class NamedCache {
080    private String name; 
081    private List<CacheEntry> list = new ArrayList<CacheEntry>(); // persistent entries
082    private Map<String, CacheEntry> map = new HashMap<String, CacheEntry>();
083  }
084  
085
086  private Object lock;
087  private String folder;
088  private Map<String, NamedCache> caches = new HashMap<String, NamedCache>();
089  
090  // use lock from the context
091  public TerminologyCache(Object lock, String folder) throws FileNotFoundException, IOException, FHIRException {
092    super();
093    this.lock = lock;
094    this.folder = folder;
095    if (folder != null)
096      load();
097  }
098  
099  public CacheToken generateValidationToken(Coding code, ValueSet vs) {
100    CacheToken ct = new CacheToken();
101    if (code.hasSystem())
102      ct.name = getNameForSystem(code.getSystem());
103    else
104      ct.name = NAME_FOR_NO_SYSTEM;
105    JsonParser json = new JsonParser();
106    json.setOutputStyle(OutputStyle.PRETTY);
107    ValueSet vsc = getVSEssense(vs);
108    try {
109      ct.request = "{\"code\" : "+json.composeString(code, "code")+", \"valueSet\" :"+(vsc == null ? "null" : json.composeString(vsc))+"}";
110    } catch (IOException e) {
111      throw new Error(e);
112    }
113    ct.key = String.valueOf(hashNWS(ct.request));
114    return ct;
115  }
116
117  public CacheToken generateValidationToken(CodeableConcept code, ValueSet vs) {
118    CacheToken ct = new CacheToken();
119    for (Coding c : code.getCoding()) {
120      if (c.hasSystem())
121        ct.setName(getNameForSystem(c.getSystem()));
122    }
123    JsonParser json = new JsonParser();
124    json.setOutputStyle(OutputStyle.PRETTY);
125    ValueSet vsc = getVSEssense(vs);
126    try {
127      ct.request = "{\"code\" : "+json.composeString(code, "codeableConcept")+", \"valueSet\" :"+json.composeString(vsc)+"}";
128    } catch (IOException e) {
129      throw new Error(e);
130    }
131    ct.key = String.valueOf(hashNWS(ct.request));
132    return ct;
133  }
134  
135  public ValueSet getVSEssense(ValueSet vs) {
136    if (vs == null)
137      return null;
138    ValueSet vsc = new ValueSet();
139    vsc.setCompose(vs.getCompose());
140    if (vs.hasExpansion()) {
141      vsc.getExpansion().getParameter().addAll(vs.getExpansion().getParameter());
142      vsc.getExpansion().getContains().addAll(vs.getExpansion().getContains());
143    }
144    return vsc;
145  }
146
147  public CacheToken generateExpandToken(ValueSet vs, boolean heirarchical) {
148    CacheToken ct = new CacheToken();
149    ValueSet vsc = getVSEssense(vs);
150    for (ConceptSetComponent inc : vs.getCompose().getInclude())
151      if (inc.hasSystem())
152        ct.setName(getNameForSystem(inc.getSystem()));
153    for (ConceptSetComponent inc : vs.getCompose().getExclude())
154      if (inc.hasSystem())
155        ct.setName(getNameForSystem(inc.getSystem()));
156    for (ValueSetExpansionContainsComponent inc : vs.getExpansion().getContains())
157      if (inc.hasSystem())
158        ct.setName(getNameForSystem(inc.getSystem()));
159    JsonParser json = new JsonParser();
160    json.setOutputStyle(OutputStyle.PRETTY);
161    try {
162      ct.request = "{\"hierarchical\" : "+(heirarchical ? "true" : "false")+", \"valueSet\" :"+json.composeString(vsc)+"}\r\n";
163    } catch (IOException e) {
164      throw new Error(e);
165    }
166    ct.key = String.valueOf(hashNWS(ct.request));
167    return ct;
168  }
169
170  private String getNameForSystem(String system) {
171    if (system.equals("http://snomed.info/sct"))
172      return "snomed";
173    if (system.equals("http://www.nlm.nih.gov/research/umls/rxnorm"))
174      return "rxnorm";
175    if (system.equals("http://loinc.org"))
176      return "loinc";
177    if (system.equals("http://unitsofmeasure.org"))
178      return "ucum";
179    if (system.startsWith("http://hl7.org/fhir/sid/"))
180      return system.substring(24).replace("/", "");
181    if (system.startsWith("urn:iso:std:iso:"))
182      return "iso"+system.substring(16).replace(":", "");
183    if (system.startsWith("http://terminology.hl7.org/CodeSystem/"))
184      return system.substring(38).replace("/", "");
185    if (system.startsWith("http://hl7.org/fhir/"))
186      return system.substring(20).replace("/", "");
187    if (system.equals("urn:ietf:bcp:47"))
188      return "lang";
189    if (system.equals("urn:ietf:bcp:13"))
190      return "mimetypes";
191    if (system.equals("urn:iso:std:iso:11073:10101"))
192      return "11073";
193    if (system.equals("http://dicom.nema.org/resources/ontology/DCM"))
194      return "dicom";
195    return system.replace("/", "_").replace(":", "_");
196  }
197
198  public NamedCache getNamedCache(CacheToken cacheToken) {
199    NamedCache nc = caches.get(cacheToken.name);
200    if (nc == null) {
201      nc = new NamedCache();
202      nc.name = cacheToken.name;
203      caches.put(nc.name, nc);
204    }
205    return nc;
206  }
207  
208  public ValueSetExpansionOutcome getExpansion(CacheToken cacheToken) {
209    synchronized (lock) {
210      NamedCache nc = getNamedCache(cacheToken);
211      CacheEntry e = nc.map.get(cacheToken.key);
212      if (e == null)
213        return null;
214      else
215        return e.e;
216    }
217  }
218
219  public void cacheExpansion(CacheToken cacheToken, ValueSetExpansionOutcome res, boolean persistent) {
220    synchronized (lock) {      
221      NamedCache nc = getNamedCache(cacheToken);
222      CacheEntry e = new CacheEntry();
223      e.request = cacheToken.request;
224      e.persistent = persistent;
225      e.e = res;
226      store(cacheToken, persistent, nc, e);
227    }    
228  }
229
230  public void store(CacheToken cacheToken, boolean persistent, NamedCache nc, CacheEntry e) {
231    boolean n = nc.map.containsKey(cacheToken.key);
232    nc.map.put(cacheToken.key, e);
233    if (persistent) {
234      if (n) {
235        for (int i = nc.list.size()- 1; i>= 0; i--) {
236          if (nc.list.get(i).request.equals(e.request)) {
237            nc.list.remove(i);
238          }
239        }
240      }
241      nc.list.add(e);
242      save(nc);  
243    }
244  }
245
246  public ValidationResult getValidation(CacheToken cacheToken) {
247    synchronized (lock) {
248      NamedCache nc = getNamedCache(cacheToken);
249      CacheEntry e = nc.map.get(cacheToken.key);
250      if (e == null)
251        return null;
252      else
253        return e.v;
254    }
255  }
256
257  public void cacheValidation(CacheToken cacheToken, ValidationResult res, boolean persistent) {
258    synchronized (lock) {      
259      NamedCache nc = getNamedCache(cacheToken);
260      CacheEntry e = new CacheEntry();
261      e.request = cacheToken.request;
262      e.persistent = persistent;
263      e.v = res;
264      store(cacheToken, persistent, nc, e);
265    }    
266  }
267
268  
269  // persistence
270  
271  public void save() {
272    
273  }
274  
275  private void save(NamedCache nc) {
276    if (folder == null)
277      return;
278    
279    try {
280      FileWriter sw = new FileWriter(new File(Utilities.path(folder, nc.name+".cache")));
281      sw.write(ENTRY_MARKER+"\r\n");
282      JsonParser json = new JsonParser();
283      json.setOutputStyle(OutputStyle.PRETTY);
284      for (CacheEntry ce : nc.list) {
285        sw.write(ce.request);
286        sw.write(BREAK+"\r\n");
287        if (ce.e != null) {
288          sw.write("e: {\r\n");
289          if (ce.e.getValueset() != null)
290            sw.write("  \"valueSet\" : "+json.composeString(ce.e.getValueset())+",\r\n");
291          sw.write("  \"error\" : \""+Utilities.escapeJson(ce.e.getError())+"\"\r\n}\r\n");
292        } else {
293          sw.write("v: {\r\n");
294          sw.write("  \"display\" : \""+Utilities.escapeJson(ce.v.getDisplay())+"\",\r\n");
295          sw.write("  \"severity\" : "+(ce.v.getSeverity() == null ? "null" : "\""+ce.v.getSeverity().toCode()+"\"")+",\r\n");
296          sw.write("  \"error\" : \""+Utilities.escapeJson(ce.v.getMessage())+"\"\r\n}\r\n");
297        }
298        sw.write(ENTRY_MARKER+"\r\n");
299      }      
300      sw.close();
301    } catch (Exception e) {
302      System.out.println("error saving "+nc.name+": "+e.getMessage());
303    }
304  }
305
306  private void load() throws FHIRException {
307    for (String fn : new File(folder).list()) {
308      if (fn.endsWith(".cache") && !fn.equals("validation.cache")) {
309        try {
310          //  System.out.println("Load "+fn);
311          String title = fn.substring(0, fn.lastIndexOf("."));
312          NamedCache nc = new NamedCache();
313          nc.name = title;
314          caches.put(title, nc);
315          String src = TextFile.fileToString(Utilities.path(folder, fn));
316          int i = src.indexOf(ENTRY_MARKER); 
317          while (i > -1) {
318            String s = src.substring(0, i);
319            src = src.substring(i+ENTRY_MARKER.length()+1);
320            i = src.indexOf(ENTRY_MARKER);
321            if (!Utilities.noString(s)) {
322              int j = s.indexOf(BREAK);
323              String q = s.substring(0, j);
324              String p = s.substring(j+BREAK.length()+1).trim();
325              CacheEntry ce = new CacheEntry();
326              ce.persistent = true;
327              ce.request = q;
328              boolean e = p.charAt(0) == 'e';
329              p = p.substring(3);
330              JsonObject o = (JsonObject) new com.google.gson.JsonParser().parse(p);
331              String error = loadJS(o.get("error"));
332              if (e) {
333                if (o.has("valueSet"))
334                  ce.e = new ValueSetExpansionOutcome((ValueSet) new JsonParser().parse(o.getAsJsonObject("valueSet")), error, TerminologyServiceErrorClass.UNKNOWN);
335                else
336                  ce.e = new ValueSetExpansionOutcome(error, TerminologyServiceErrorClass.UNKNOWN);
337              } else {
338                IssueSeverity severity = o.get("severity") instanceof JsonNull ? null :  IssueSeverity.fromCode(o.get("severity").getAsString());
339                String display = loadJS(o.get("display"));
340                ce.v = new ValidationResult(severity, error, new ConceptDefinitionComponent().setDisplay(display));
341              }
342              nc.map.put(String.valueOf(hashNWS(ce.request)), ce);
343              nc.list.add(ce);
344            }
345          }        
346        } catch (Exception e) {
347          throw new FHIRException("Error loading "+fn+": "+e.getMessage(), e);
348        }
349      }
350    }
351  }
352  
353  private String loadJS(JsonElement e) {
354    if (e == null)
355      return null;
356    if (!(e instanceof JsonPrimitive))
357      return null;
358    String s = e.getAsString();
359    if ("".equals(s))
360      return null;
361    return s;
362  }
363
364  private String hashNWS(String s) {
365    return String.valueOf(s.replace("\r", "").replace("\n", "").replace(" ", "").hashCode());
366  }
367
368  // management
369  
370  public TerminologyCache copy() {
371    // TODO Auto-generated method stub
372    return null;
373  }
374  
375  public String summary(ValueSet vs) {
376    if (vs == null)
377      return "null";
378    
379    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
380    for (ConceptSetComponent cc : vs.getCompose().getInclude())
381      b.append("Include "+getIncSummary(cc));
382    for (ConceptSetComponent cc : vs.getCompose().getExclude())
383      b.append("Exclude "+getIncSummary(cc));
384    return b.toString();
385  }
386
387  private String getIncSummary(ConceptSetComponent cc) {
388    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
389    for (UriType vs : cc.getValueSet())
390      b.append(vs.asStringValue());
391    String vsd = b.length() > 0 ? " where the codes are in the value sets ("+b.toString()+")" : "";
392    String system = cc.getSystem();
393    if (cc.hasConcept())
394      return Integer.toString(cc.getConcept().size())+" codes from "+system+vsd;
395    if (cc.hasFilter()) {
396      String s = "";
397      for (ConceptSetFilterComponent f : cc.getFilter()) {
398        if (!Utilities.noString(s))
399          s = s + " & ";
400        s = s + f.getProperty()+" "+f.getOp().toCode()+" "+f.getValue();
401      }
402      return "from "+system+" where "+s+vsd;
403    }
404    return "All codes from "+system+vsd;
405  }
406
407  public String summary(Coding code) {
408    return code.getSystem()+"#"+code.getCode()+": \""+code.getDisplay()+"\"";
409  }
410
411
412  public String summary(CodeableConcept code) {
413    StringBuilder b = new StringBuilder();
414    b.append("{");
415    boolean first = true;
416    for (Coding c : code.getCoding()) {
417      if (first) first = false; else b.append(",");
418      b.append(summary(c));
419    }
420    b.append("}: \"");
421    b.append(code.getText());
422    b.append("\"");
423    return b.toString();
424  }
425  
426}