EMERSE Integration and Indexing Guide

import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.CharBuffer;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.function.Function;

/**
 * This directly hits Solr's REST API using only native Java.  See:
 *
 * http://lucene.apache.org/solr/guide/7_3/indexing-and-basic-data-operations.html
 *
 * For more details and more production-ready options.
 *
 * To run this file, compile it to a .class file, then run the class file by its class name, the
 * name without the extension:
 * <pre>
 *     javac Index.java
 *     java Index path-to-data-directory
 * </pre>
 * <p>
 *
 * The data directory should contain a hierarchy of files.  The first level of directories
 * should name the MRN number, the second level should name the source key, and the files
 * in those directories are the files that are indexed.  For example:
 *
 * 21-77/source1/visit1.html
 * 21-77/source1/visit2.html
 * 21-77/source2/visit3.html
 * 18-32/source3/visit1.html
 *
 * 21-77 is the MRN number, source1 names the source in the database, and visit1.txt
 * is just a file name used in the ID of the solr document created.  (The id is the
 * whole path from the data directory.)
 */
public class Index {

	private static final Function<String, String> NOW_AS_ISO;
	private static final int SOLR_PORT;
	private static final String INDEX_NAME;
	private static Writer writer;

	private static void printUsage() {
		System.err.println("Usage: java [ -DsolrPort=8983 ] ");
		System.err.println("            [ -DindexName=documents ]");
		System.err.println("            Index <data-directory>\n");

		System.err.println("       Options must be before the class file.  ");
		System.err.println("       Defaults are as listed.\n");

		System.err.println("       The first argument after the class file must be ");
		System.err.println("       the directory data is stored in.");
		System.err.println("       The first level of directories names the MRN,");
		System.err.println("       the second level of directories are the source keys,");
		System.err.println("       and the filename is included as part of the document id.");
		System.err.println("       So radiology files for patient 27-77 would appear like:");

		System.err.println("           27-77/source3/visit1.html");
		System.err.println("           27-77/source3/visit2.html");
		System.err.println("           27-77/source3/report1.html");
		System.err.println("           etc.\n");

		System.err.println("       (Recall that the radiology source has id \"source3\".)\n");

		System.err.println("       Each file can have a related properties file:\n");

		System.err.println("           27-77/Radiology/visit1.properties");
		System.err.println("           27-77/Radiology/visit2.properties");
		System.err.println("           27-77/Radiology/report1.properties");
		System.err.println("           etc.\n");

		System.err.println("       which populates additional document fields, such as");
		System.err.println("       document type (DOC_TYPE) or department (DEPT):\n");

		System.err.println("           DOC_TYPE=Discharge note");
		System.err.println("           DEPT=Internal medicine");


		System.exit(1);
	}

	static {
		SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
		formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
		NOW_AS_ISO = k -> formatter.format(new Date());
		SOLR_PORT = Integer.parseInt(System.getProperty("solrPort", "8983"));
		INDEX_NAME = System.getProperty("indexName", "documents");
	}

	public static void main(String[] args) {
		if (args.length != 1) {
			printUsage();
		}
		try {
			importData(args[0]);
		} catch (Exception e) {
			e.printStackTrace();
			System.err.println("\n\nEncountered error!\n\n");
			printUsage();
		}
	}

	private static void importData(String baseDir) throws Exception {
		@SuppressWarnings("unchecked")
		Map<String, String> p = (Map) new Properties();


		File[] patients = new File(baseDir).listFiles(File::isDirectory);
		if (patients == null) {
			return;
		}
		for (File patient : patients) {
			String mrn = patient.getName();
			File[] sources = patient.listFiles(File::isDirectory);
			if (sources == null) {
				return;
			}
			for (File source : sources) {
				String sourceKey = source.getName();
				File[] files =
						source.listFiles(f -> f.isFile() && !f.getName().endsWith("properties"));
				if (files == null) {
					continue;
				}
				for (File file : files) {
					String docId =
							mrn + File.separator + sourceKey + File.separator + file.getName();

					loadProperties((Properties) (Map) p, file);

					p.put("ID", docId);
					p.put("MRN", mrn);
					p.put("SOURCE", sourceKey);
					p.put("RPT_TEXT", getFileContents(file));
					p.computeIfAbsent("ENCOUNTER_DATE", NOW_AS_ISO);

					indexDocument(p, docId);
				}
			}
		}
	}

	private static void indexDocument(Map<String, String> map, String docId) throws Exception {
		HttpURLConnection connection = (HttpURLConnection) new URL(
				"http",
				"localhost",
				SOLR_PORT,
				"/solr/" + INDEX_NAME + "/update/json/docs").openConnection();
		connection.setRequestMethod("POST");
		connection.setRequestProperty("Content-Type", "application/json");
		connection.setDoOutput(true);
		writeMap(map, connection.getOutputStream());
		int status = connection.getResponseCode();
		System.out.println("Update to document " + docId + " returned status: " + status);
		if (status / 100 != 0) {
			try (Reader r = new InputStreamReader(connection.getInputStream())) {
				CharBuffer cbuf = CharBuffer.allocate(1000);
				while (r.read(cbuf) != -1) {
					System.err.append(cbuf);
					cbuf.clear();
				}
				System.err.flush();
			}
		}
	}


	private static void loadProperties(Properties p, File file) throws IOException {
		p.clear();

		String propFilename = file.getName();
		propFilename = propFilename.substring(0, propFilename.indexOf('.')) + ".properties";
		File propertiesFile = new File(file.getParentFile(), propFilename);
		if (propertiesFile.exists()) {
			p.load(new FileReader(propertiesFile));
		}
	}

	private static String getFileContents(File file) throws IOException {
		try (FileReader reader = new FileReader(file)) {
			StringBuilder sb = new StringBuilder();
			char[] buffy = new char[1000];
			int buffySize;
			while ((buffySize = reader.read(buffy)) != -1) {
				sb.append(buffy, 0, buffySize);
			}
			return sb.toString();
		}
	}

	private static void writeMap(
			Map<String, String> map, OutputStream outputStream) throws Exception {
		writer = new OutputStreamWriter(outputStream);
		writeMap(map);
	}


	private static void writeMap(Map<String, String> map) throws IOException {
		writer.write("{\n");
		Iterator<String> iterator = map.keySet().iterator();
		while (iterator.hasNext()) {
			String key = iterator.next();
			writer.write("  \"");
			writeStringEscaped(key);
			writer.write("\": \"");
			writeStringEscaped(map.get(key));
			writer.write('"');
			if (iterator.hasNext()) {
				writer.write(',');
			}
			writer.write('\n');
		}
		writer.write("}\n");
	}

	private static void writeStringEscaped(String s) throws IOException {
		for (int i = 0; i < s.length(); i++) {
			char c = s.charAt(i);
			int p = Character.codePointAt(s, i);
			if (p <= 31 || c == '"' || c == '\\') {
				writer.write(String.format("\\u%04x", p));
			} else {
				writer.write(c);
			}
		}
	}
}