pgroth$hbase-rdf
throws IOException {//we expect the pattern in the order SPOC	
			retrievalInit();
			long start = System.currentTimeMillis();
			byte[] startKey = buildRangeScanKeyFromQuad(quad);
			long keyBuildOverhead = System.currentTimeMillis()-start;
			long startSearch = System.currentTimeMillis();
			ArrayList<Get> batchIdGets = doRangeScan(startKey);
			start = System.currentTimeMillis();
			Result[] id2StringResults = doBatchId2String(batchIdGets);
			System.out.println("Search time: "+searchTime+"; Id2StringOverhead: "+id2StringOverhead+"; String2IdOverhead: "
									+string2IdOverhead+"; KeyBuildOverhead: "+keyBuildOverhead);
			updateId2ValueMap(id2StringResults);			
			return buildSPOCOrderResults();
		
			System.err.println("Bound variable numerical not in expected range: "+e.getMessage());
			return null;
void
void buildPattern2TableHashMap()
throws IOException {//we expect the pattern in the order SPOC
		
			id2ValueMap.clear();
			quadResults.clear();
			boundElements.clear();
			id2StringOverhead = 0;
			byte[] startKey = buildKey(quad);
			if (startKey == null) {
				return new ArrayList<ArrayList<Value>>();
void
void buildHashMap()
pgroth$hbase-rdf
Value quad
byte[]
byte[] buildRangeScanKeyFromQuad(Value quad)
Value quad
byte[]
byte[] buildKey(Value quad)
pgroth$hbase-rdf
Job j1
void
void retrieveTripleToResourceCounters(Job j1)
Job j1
void
void retrieveCounters(Job j1)
https://github.com/pgroth/hbase-rdf/blob/cbf0cac3f03c8dffbc415eab1421234dfc3ead24/src%2Fmain%2Fjava%2Fnl%2Fvu%2Fdatalayer%2Fhbase%2Fbulkload%2FBulkLoad.java
https://api.github.com/repos/pgroth/hbase-rdf/contents/src%2Fmain%2Fjava%2Fnl%2Fvu%2Fdatalayer%2Fhbase%2Fbulkload%2FBulkLoad.java?ref=cbf0cac3f03c8dffbc415eab1421234dfc3ead24
src/main/java/nl/vu/datalayer/hbase/bulkload/BulkLoad.java
import nl.vu.datalayer.hbase.id.HBaseValue;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
	//public static SortedMap<Short, Long> sufixCounters;

	private static Path input;

	private static int inputEstimateSize;

	private static String outputPath;

	private static boolean onlyTriples;

	private static NativeJavaConnection con;
	/*public static void main(String[] args) {
		try {
		Path input = new Path(args[0]);
		String outputPath = args[2];
		Path resourceIds = new Path(outputPath+TripleToResource.RESOURCE_IDS_DIR);
		Path convertedTripletsPath = new Path(outputPath+ResourceToTriple.TEMP_TRIPLETS_DIR);
		
		Path string2IdInput = new Path(outputPath+TripleToResource.ID2STRING_DIR);
		Path string2IdOutput = new Path(outputPath+StringIdAssoc.STRING2ID_DIR);
		
		HBaseConnection con = new HBaseConnection(HBaseConnection.JAVA_API);
		HBPrefixMatchSchema.createCounterTable(con.getAdmin());
		
		long start = System.currentTimeMillis();
		
		Job j1 = createTripleToResourceJob(input, resourceIds, Integer.parseInt(args[1]));
		j1.getConfiguration().set("outputPath", outputPath);
		j1.waitForCompletion(true);
		
		long firstJob = System.currentTimeMillis()-start;
		System.out.println("First pass finished in "+firstJob+" ms");
		
		//retrieve counter values and build a SortedMap
		retrieveCounters(j1);
		long startPartition = HBPrefixMatchSchema.updateLastCounter(tripleToResourceReduceTasks, con.getConfiguration())+1;
		}
		catch (Exception e) {
			e.printStackTrace();
		}
	}*/
	
	/**
	 * @param args
	 */
	public static void main(String[] args) {
		try {
			if (args.length != 5){
				System.out.println("Usage: bulkLoad <inputPath> <inputSizeEstimate in MB> <outputPath> <schemaSuffix> <onlyTriples(true/false)>");
				return;
			}		
			initializeLocalVariables(args); 
			
			Path convertedTripletsPath = new Path(outputPath+ResourceToTriple.TEMP_TRIPLETS_DIR);
			Path idStringAssocInput = new Path(outputPath+"/"+TripleToResource.ID2STRING_DIR);
			
			con = (NativeJavaConnection)HBaseConnection.create(HBaseConnection.NATIVE_JAVA);
			HBPrefixMatchSchema prefMatchSchema = new HBPrefixMatchSchema(con, schemaSuffix);
			prefMatchSchema.createCounterTable(con.getAdmin());
			
			long startPartition = runIdGenerationJobs(convertedTripletsPath, idStringAssocInput);
		
			//create all tables containing PrefixMatch schema ----------------------
			System.out.println(totalStringCount+" : "+numericalCount);
					
			prefMatchSchema.setTableSplitInfo(totalStringCount, numericalCount, 
					tripleToResourceReduceTasks, startPartition, onlyTriples);
			prefMatchSchema.create();
			
			LoadIncrementalHFiles bulkLoad = new LoadIncrementalHFiles(con.getConfiguration());
			
			bulkLoadIdStringMappingTables(idStringAssocInput, bulkLoad);		
			bulkLoadQuadTables(convertedTripletsPath, bulkLoad);
			
			con.close();		
		} catch (IOException e) {
			e.printStackTrace();
		}
		catch(Exception e){
			e.printStackTrace();
		}
	}

	private static void bulkLoadQuadTables(Path convertedTripletsPath, LoadIncrementalHFiles bulkLoad) throws Exception, IOException, InterruptedException, ClassNotFoundException {
		Path spocPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.SPOC]);
		Path pocsPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.POCS]);
		Path ospcPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.OSPC]);
		
		//SPOC--------------------
		Job j5 = createPrefixMatchJob(con, convertedTripletsPath, spocPath, HBPrefixMatchSchema.SPOC, PrefixMatch.PrefixMatchSPOCMapper.class);
		j5.waitForCompletion(true);
		
		doBulkLoad(bulkLoad, spocPath, currentTable, con);

		if (onlyTriples == false){
			bulkLoadQuadOnlyTables(convertedTripletsPath, bulkLoad);
		}
		
		//OSPC---------------------------
		Job j10 = createPrefixMatchJob(con, convertedTripletsPath, ospcPath, HBPrefixMatchSchema.OSPC, PrefixMatch.PrefixMatchOSPCMapper.class);
		j10.waitForCompletion(true);
		
		doBulkLoad(bulkLoad, ospcPath, currentTable, con);
		
		//POCS---------------------
		Job j6 = createPrefixMatchJob(con, convertedTripletsPath, pocsPath, HBPrefixMatchSchema.POCS, PrefixMatch.PrefixMatchPOCSMapper.class);
		j6.waitForCompletion(true);
		
		doBulkLoad(bulkLoad, pocsPath, currentTable, con);
	}

	private static void bulkLoadQuadOnlyTables(Path convertedTripletsPath, LoadIncrementalHFiles bulkLoad) throws Exception, IOException, InterruptedException, ClassNotFoundException {
		Path cspoPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.CSPO]);
		Path cpsoPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.CPSO]);
		Path ocspPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.OCSP]);
		
		//CSPO----------------------
		Job j7 = createPrefixMatchJob(con, convertedTripletsPath, cspoPath, HBPrefixMatchSchema.CSPO, PrefixMatch.PrefixMatchCSPOMapper.class);
		j7.waitForCompletion(true);
		
		doBulkLoad(bulkLoad, cspoPath, currentTable, con);
		
		//CPSO-------------------------
		Job j8 = createPrefixMatchJob(con, convertedTripletsPath, cpsoPath, HBPrefixMatchSchema.CPSO, PrefixMatch.PrefixMatchCPSOMapper.class);
		j8.waitForCompletion(true);
		
		doBulkLoad(bulkLoad, cpsoPath, currentTable, con);
		
		//OCSP--------------------------			
		Job j9 = createPrefixMatchJob(con, convertedTripletsPath, ocspPath, HBPrefixMatchSchema.OCSP, PrefixMatch.PrefixMatchOCSPMapper.class);
		j9.waitForCompletion(true);
		
		doBulkLoad(bulkLoad, ocspPath, currentTable, con);
	}

	final private static void bulkLoadIdStringMappingTables(Path idStringAssocInput, LoadIncrementalHFiles bulkLoad) throws Exception, IOException, InterruptedException, ClassNotFoundException {
		Path string2IdOutput = new Path(outputPath+StringIdAssoc.STRING2ID_DIR);
		Path id2StringOutput = new Path(outputPath+StringIdAssoc.ID2STRING_DIR);
		
		//String2Id PASS----------------------------------------------		
		Job j3 = createString2IdJob(con, idStringAssocInput, string2IdOutput);
		j3.waitForCompletion(true);
		
		//string2Id = con.getTable(HBPrefixMatchSchema.STRING2ID+schemaSuffix);
		doBulkLoad(bulkLoad, string2IdOutput, string2Id, con);
		
		System.out.println("Finished bulk load for String2Id table");//=====================//=============
		
		Job j4 = createId2StringJob(con, idStringAssocInput, id2StringOutput);
		j4.waitForCompletion(true);
		
		doBulkLoad(bulkLoad, id2StringOutput, id2String, con);		
		System.out.println("Finished bulk load for Id2String table");//=====================//==================================
	}

	final private static long runIdGenerationJobs(Path convertedTripletsPath, Path idStringAssocInput) throws IOException, InterruptedException, ClassNotFoundException {		
		Path resourceIds = new Path(outputPath+"/"+TripleToResource.RESOURCE_IDS_DIR);
		
		runTripleToResourceJob(idStringAssocInput, resourceIds);
		long startPartition = HBPrefixMatchSchema.updateLastCounter(tripleToResourceReduceTasks, con.getConfiguration(), schemaSuffix)+1;
		//long startPartition = HBPrefixMatchSchema.getLastCounter(con.getConfiguration())+1;
		
		//SECOND PASS-----------------------------------------------	
		Job j2 = createResourceToTripleJob(resourceIds, convertedTripletsPath);	
		j2.waitForCompletion(true);
		System.out.println("Second pass finished");
		
		//read counters from file
		//buildCountersFromFile();
		
		return startPartition;
	}

	private static void runTripleToResourceJob(Path idStringAssocInput, Path resourceIds) throws IOException, InterruptedException, ClassNotFoundException {
		long start = System.currentTimeMillis();
		Job j1 = createTripleToResourceJob(input, resourceIds, inputEstimateSize);
		j1.getConfiguration().set("outputPath", outputPath);
		j1.getConfiguration().set("schemaSuffix", schemaSuffix);
		j1.waitForCompletion(true);
		
		//move side effect files out of TripleToResource output directory
		moveIdStringAssocDirectory(resourceIds, idStringAssocInput);
		
		long firstJob = System.currentTimeMillis()-start;
		System.out.println("First pass finished in "+firstJob+" ms");
		retrieveTripleToResourceCounters(j1);
	}

	private static void initializeLocalVariables(String[] args) {
		input = new Path(args[0]);
		inputEstimateSize = Integer.parseInt(args[1]);
		outputPath = args[2];
		schemaSuffix = args[3];
		onlyTriples = Boolean.parseBoolean(args[4]);
	}


	public static Job createTripleToResourceJob(Path input, Path output, int inputSizeEstimate) throws IOException{
		//tripleToResourceReduceTasks = (int)(Math.ceil((double)totalNumberOfElements/Math.pow(2.0, 24.0)) * LOAD_BALANCER_FACTOR);
		tripleToResourceReduceTasks = 2;//TODO
		j.setMapOutputKeyClass(HBaseValue.class);
	public static void retrieveTripleToResourceCounters(Job j1) throws IOException{
		//sufixCounters = new TreeMap<Short, Long>();
		/*CounterGroup histogramGroup = counters.getGroup(TripleToResource.TripleToResourceReducer.HISTOGRAM_GROUP);
		}*/		
		/*for (Map.Entry<Short, Long> entry : sufixCounters.entrySet()) {
		}*/
		//sufixCounters = new TreeMap();
		/*String line;
		System.out.println(totalCounter);*/
	private static void doBulkLoad(LoadIncrementalHFiles bulkLoad, Path dir, HTableInterface table, NativeJavaConnection con) throws InterruptedException{
		boolean timedOut = false;
				
			System.out.println("Bulk load taking longer than usual: sleeping for 10 minutes before proceeding to the next table..");
			timedOut = true;
		if (timedOut){//the BulkLoad took too long - which means the cluster is unbalanced
			try {
				con.getAdmin().split(table.getTableName());
				Thread.sleep(300000);//TODO find a better method to do these splits
			} catch (IOException e) {
				e.printStackTrace();
		}
	}
	public static void moveIdStringAssocDirectory(Path resourceIds, Path id2StringInput) throws IOException{
		Configuration conf = new Configuration();
		FileSystem fs = FileSystem.get(conf);
		Path source = new Path(resourceIds, TripleToResource.ID2STRING_DIR);
		FileUtil.copy(fs, source, fs, id2StringInput, true, false, conf);
	
String[] args
src/main/java/nl/vu/datalayer/hbase/bulkload/BulkLoad.java
try {
		Path input = new Path(args[0]);
		String outputPath = args[2];
		Path resourceIds = new Path(outputPath+TripleToResource.RESOURCE_IDS_DIR);
		Path convertedTripletsPath = new Path(outputPath+ResourceToTriple.TEMP_TRIPLETS_DIR);
		
		Path string2IdInput = new Path(outputPath+TripleToResource.ID2STRING_DIR);
		Path string2IdOutput = new Path(outputPath+StringIdAssoc.STRING2ID_DIR);
		
		HBaseConnection con = new HBaseConnection(HBaseConnection.JAVA_API);
		HBPrefixMatchSchema.createCounterTable(con.getAdmin());
		
		long start = System.currentTimeMillis();
		
		Job j1 = createTripleToResourceJob(input, resourceIds, Integer.parseInt(args[1]));
		j1.getConfiguration().set("outputPath", outputPath);
		j1.waitForCompletion(true);
		
		long firstJob = System.currentTimeMillis()-start;
		System.out.println("First pass finished in "+firstJob+" ms");
		
		//retrieve counter values and build a SortedMap
		retrieveCounters(j1);
		long startPartition = HBPrefixMatchSchema.updateLastCounter(tripleToResourceReduceTasks, con.getConfiguration())+1;
void main(String[] args)
String[] args
src/main/java/nl/vu/datalayer/hbase/bulkload/BulkLoad.java
try {
			if (args.length != 5){
				System.out.println("Usage: bulkLoad <inputPath> <inputSizeEstimate in MB> <outputPath> <schemaSuffix> <onlyTriples(true/false)>");
				return;
void main(String[] args)
String[] args
src/main/java/nl/vu/datalayer/hbase/bulkload/BulkLoad.java
input = new Path(args[0]);
		inputEstimateSize = Integer.parseInt(args[1]);
		outputPath = args[2];
		schemaSuffix = args[3];
		onlyTriples = Boolean.parseBoolean(args[4]);
void initializeLocalVariables(String[] args)
import org.apache.hadoop.io.Text;
	public static SortedMap<Short, Long> sufixCounters;
	public static Job createTripleToResourceJob(Path input, Path output, int inputSizeEstimate) throws IOException{
		tripleToResourceReduceTasks = (int)(Math.ceil((double)totalNumberOfElements/Math.pow(2.0, 24.0)) * LOAD_BALANCER_FACTOR);
		j.setMapOutputKeyClass(Text.class);
	public static void retrieveCounters(Job j1) throws IOException{
		sufixCounters = new TreeMap<Short, Long>();
		CounterGroup histogramGroup = counters.getGroup(TripleToResource.TripleToResourceReducer.HISTOGRAM_GROUP);
		}		
		for (Map.Entry<Short, Long> entry : sufixCounters.entrySet()) {
		}
		sufixCounters = new TreeMap();
		String line;
		System.out.println(totalCounter);
	private static void doBulkLoad(LoadIncrementalHFiles bulkLoad, Path dir, HTableInterface table) throws InterruptedException{
			System.out.println("Bulk load taking longer than usual: sleeping for 5 minutes before proceeding to the next table..");
	}
	
	/*public static void main(String[] args) {
		try {
		Path input = new Path(args[0]);
		String outputPath = args[2];
		Path resourceIds = new Path(outputPath+TripleToResource.RESOURCE_IDS_DIR);
		Path convertedTripletsPath = new Path(outputPath+ResourceToTriple.TEMP_TRIPLETS_DIR);
		
		Path string2IdInput = new Path(outputPath+TripleToResource.ID2STRING_DIR);
		Path string2IdOutput = new Path(outputPath+StringIdAssoc.STRING2ID_DIR);
		
		HBaseConnection con = new HBaseConnection(HBaseConnection.JAVA_API);
		HBPrefixMatchSchema.createCounterTable(con.getAdmin());
		
		long start = System.currentTimeMillis();
		
		Job j1 = createTripleToResourceJob(input, resourceIds, Integer.parseInt(args[1]));
		j1.getConfiguration().set("outputPath", outputPath);
		j1.waitForCompletion(true);
		long firstJob = System.currentTimeMillis()-start;
		System.out.println("First pass finished in "+firstJob+" ms");
		
		//retrieve counter values and build a SortedMap
		retrieveCounters(j1);
		long startPartition = HBPrefixMatchSchema.updateLastCounter(tripleToResourceReduceTasks, con.getConfiguration())+1;
		}
		catch (Exception e) {
			e.printStackTrace();
		}
	}*/
	
	/**
	 * @param args
	 */
	public static void main(String[] args) {
		try {
			if (args.length != 5){
				System.out.println("Usage: bulkLoad <inputPath> <inputSizeEstimate in MB> <outputPath> <schemaSuffix> <onlyTriples(true/false)>");
				return;
			
			Path input = new Path(args[0]);
			String outputPath = args[2];
			schemaSuffix = args[3];
			boolean onlyTriples = Boolean.parseBoolean(args[4]); 
			
			Path resourceIds = new Path(outputPath+TripleToResource.RESOURCE_IDS_DIR);
			Path convertedTripletsPath = new Path(outputPath+ResourceToTriple.TEMP_TRIPLETS_DIR);
			
			Path string2IdInput = new Path(outputPath+TripleToResource.ID2STRING_DIR);
			Path string2IdOutput = new Path(outputPath+StringIdAssoc.STRING2ID_DIR);
			
			NativeJavaConnection con = (NativeJavaConnection)HBaseConnection.create(HBaseConnection.NATIVE_JAVA);
			HBPrefixMatchSchema prefMatchSchema = new HBPrefixMatchSchema(con, schemaSuffix);
			prefMatchSchema.createCounterTable(con.getAdmin());
			
			long start = System.currentTimeMillis();
			
			Job j1 = createTripleToResourceJob(input, resourceIds, Integer.parseInt(args[1]));//TODO
			j1.getConfiguration().set("outputPath", outputPath);
			j1.getConfiguration().set("schemaSuffix", schemaSuffix);
			j1.waitForCompletion(true);
			
			long firstJob = System.currentTimeMillis()-start;
			System.out.println("First pass finished in "+firstJob+" ms");
			
			//retrieve counter values and build a SortedMap
			retrieveCounters(j1);//TODO
			long startPartition = HBPrefixMatchSchema.updateLastCounter(tripleToResourceReduceTasks, con.getConfiguration(), schemaSuffix)+1;//TODO
			//long startPartition = HBPrefixMatchSchema.getLastCounter(con.getConfiguration())+1;
			
			
			//SECOND PASS-----------------------------------------------	
			
			Job j2 = createResourceToTripleJob(resourceIds, convertedTripletsPath);	
			j2.waitForCompletion(true);
			
			System.out.println("Second pass finished");
			
			//read counters from file
			//buildCountersFromFile();
			//create all tables containing PrefixMatch schema ----------------------
			System.out.println(totalStringCount+" : "+numericalCount);
					
			prefMatchSchema.setTableSplitInfo(totalStringCount, numericalCount, 
					tripleToResourceReduceTasks, startPartition, sufixCounters, 
					onlyTriples);
			prefMatchSchema.create();
			
			LoadIncrementalHFiles bulkLoad = new LoadIncrementalHFiles(con.getConfiguration());
			
			//String2Id PASS----------------------------------------------
					
			Job j3 = createString2IdJob(con, string2IdInput, string2IdOutput);
			j3.waitForCompletion(true);
			
			//string2Id = con.getTable(HBPrefixMatchSchema.STRING2ID);
			doBulkLoad(bulkLoad, string2IdOutput, string2Id);
			
			System.out.println("Finished bulk load for String2Id table");//=====================//=============
			
			Path id2StringOutput = new Path(outputPath+StringIdAssoc.ID2STRING_DIR);
			Job j4 = createId2StringJob(con, string2IdInput, id2StringOutput);
			j4.waitForCompletion(true);
			
			doBulkLoad(bulkLoad, id2StringOutput, id2String);
			
			System.out.println("Finished bulk load for Id2String table");//=====================//==================================
			
			Path spocPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.SPOC]);
			Path pocsPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.POCS]);
			Path ospcPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.OSPC]);
			
			//SPOC--------------------
			Job j5 = createPrefixMatchJob(con, convertedTripletsPath, spocPath, HBPrefixMatchSchema.SPOC, PrefixMatch.PrefixMatchSPOCMapper.class);
			j5.waitForCompletion(true);
			
			doBulkLoad(bulkLoad, spocPath, currentTable);
			if (onlyTriples == false){//the table with prefix context
				Path cspoPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.CSPO]);
				Path cpsoPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.CPSO]);
				Path ocspPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.OCSP]);
				
				//CSPO----------------------
				Job j7 = createPrefixMatchJob(con, convertedTripletsPath, cspoPath, HBPrefixMatchSchema.CSPO, PrefixMatch.PrefixMatchCSPOMapper.class);
				j7.waitForCompletion(true);
				
				doBulkLoad(bulkLoad, cspoPath, currentTable);
				
				//CPSO-------------------------
				Job j8 = createPrefixMatchJob(con, convertedTripletsPath, cpsoPath, HBPrefixMatchSchema.CPSO, PrefixMatch.PrefixMatchCPSOMapper.class);
				j8.waitForCompletion(true);
				
				doBulkLoad(bulkLoad, cpsoPath, currentTable);
				
				//OCSP--------------------------			
				Job j9 = createPrefixMatchJob(con, convertedTripletsPath, ocspPath, HBPrefixMatchSchema.OCSP, PrefixMatch.PrefixMatchOCSPMapper.class);
				j9.waitForCompletion(true);
				
				doBulkLoad(bulkLoad, ocspPath, currentTable);
			}
			
			//OSPC---------------------------
			Job j10 = createPrefixMatchJob(con, convertedTripletsPath, ospcPath, HBPrefixMatchSchema.OSPC, PrefixMatch.PrefixMatchOSPCMapper.class);
			j10.waitForCompletion(true);
			
			doBulkLoad(bulkLoad, ospcPath, currentTable);
			
			//POCS---------------------
			Job j6 = createPrefixMatchJob(con, convertedTripletsPath, pocsPath, HBPrefixMatchSchema.POCS, PrefixMatch.PrefixMatchPOCSMapper.class);
			j6.waitForCompletion(true);
			
			doBulkLoad(bulkLoad, pocsPath, currentTable);
			
			con.close();
			
		} catch (IOException e) {
			e.printStackTrace();
		}
		catch(Exception e){
			e.printStackTrace();
		}

String[] args
src/main/java/nl/vu/datalayer/hbase/bulkload/BulkLoad.java
try {
		Path input = new Path(args[0]);
		String outputPath = args[2];
		Path resourceIds = new Path(outputPath+TripleToResource.RESOURCE_IDS_DIR);
		Path convertedTripletsPath = new Path(outputPath+ResourceToTriple.TEMP_TRIPLETS_DIR);
		
		Path string2IdInput = new Path(outputPath+TripleToResource.ID2STRING_DIR);
		Path string2IdOutput = new Path(outputPath+StringIdAssoc.STRING2ID_DIR);
		
		HBaseConnection con = new HBaseConnection(HBaseConnection.JAVA_API);
		HBPrefixMatchSchema.createCounterTable(con.getAdmin());
		
		long start = System.currentTimeMillis();
		
		Job j1 = createTripleToResourceJob(input, resourceIds, Integer.parseInt(args[1]));
		j1.getConfiguration().set("outputPath", outputPath);
		j1.waitForCompletion(true);
		long firstJob = System.currentTimeMillis()-start;
		System.out.println("First pass finished in "+firstJob+" ms");
		
		//retrieve counter values and build a SortedMap
		retrieveCounters(j1);
		long startPartition = HBPrefixMatchSchema.updateLastCounter(tripleToResourceReduceTasks, con.getConfiguration())+1;
void main(String[] args)
String[] args
src/main/java/nl/vu/datalayer/hbase/bulkload/BulkLoad.java
try {
			if (args.length != 5){
				System.out.println("Usage: bulkLoad <inputPath> <inputSizeEstimate in MB> <outputPath> <schemaSuffix> <onlyTriples(true/false)>");
				return;
			
			Path input = new Path(args[0]);
			String outputPath = args[2];
			schemaSuffix = args[3];
			boolean onlyTriples = Boolean.parseBoolean(args[4]); 
			
			Path resourceIds = new Path(outputPath+TripleToResource.RESOURCE_IDS_DIR);
			Path convertedTripletsPath = new Path(outputPath+ResourceToTriple.TEMP_TRIPLETS_DIR);
			
			Path string2IdInput = new Path(outputPath+TripleToResource.ID2STRING_DIR);
			Path string2IdOutput = new Path(outputPath+StringIdAssoc.STRING2ID_DIR);
			
			NativeJavaConnection con = (NativeJavaConnection)HBaseConnection.create(HBaseConnection.NATIVE_JAVA);
			HBPrefixMatchSchema prefMatchSchema = new HBPrefixMatchSchema(con, schemaSuffix);
			prefMatchSchema.createCounterTable(con.getAdmin());
			
			long start = System.currentTimeMillis();
			
			Job j1 = createTripleToResourceJob(input, resourceIds, Integer.parseInt(args[1]));//TODO
			j1.getConfiguration().set("outputPath", outputPath);
			j1.getConfiguration().set("schemaSuffix", schemaSuffix);
			j1.waitForCompletion(true);
			
			long firstJob = System.currentTimeMillis()-start;
			System.out.println("First pass finished in "+firstJob+" ms");
			
			//retrieve counter values and build a SortedMap
			retrieveCounters(j1);//TODO
			long startPartition = HBPrefixMatchSchema.updateLastCounter(tripleToResourceReduceTasks, con.getConfiguration(), schemaSuffix)+1;//TODO
			//long startPartition = HBPrefixMatchSchema.getLastCounter(con.getConfiguration())+1;
			
			
			//SECOND PASS-----------------------------------------------	
			
			Job j2 = createResourceToTripleJob(resourceIds, convertedTripletsPath);	
			j2.waitForCompletion(true);
			
			System.out.println("Second pass finished");
			
			//read counters from file
			//buildCountersFromFile();
			//create all tables containing PrefixMatch schema ----------------------
			System.out.println(totalStringCount+" : "+numericalCount);
					
			prefMatchSchema.setTableSplitInfo(totalStringCount, numericalCount, 
					tripleToResourceReduceTasks, startPartition, sufixCounters, 
					onlyTriples);
			prefMatchSchema.create();
			
			LoadIncrementalHFiles bulkLoad = new LoadIncrementalHFiles(con.getConfiguration());
			
			//String2Id PASS----------------------------------------------
					
			Job j3 = createString2IdJob(con, string2IdInput, string2IdOutput);
			j3.waitForCompletion(true);
			
			//string2Id = con.getTable(HBPrefixMatchSchema.STRING2ID);
			doBulkLoad(bulkLoad, string2IdOutput, string2Id);
			
			System.out.println("Finished bulk load for String2Id table");//=====================//=============
			
			Path id2StringOutput = new Path(outputPath+StringIdAssoc.ID2STRING_DIR);
			Job j4 = createId2StringJob(con, string2IdInput, id2StringOutput);
			j4.waitForCompletion(true);
			
			doBulkLoad(bulkLoad, id2StringOutput, id2String);
			
			System.out.println("Finished bulk load for Id2String table");//=====================//==================================
			
			Path spocPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.SPOC]);
			Path pocsPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.POCS]);
			Path ospcPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.OSPC]);
			
			//SPOC--------------------
			Job j5 = createPrefixMatchJob(con, convertedTripletsPath, spocPath, HBPrefixMatchSchema.SPOC, PrefixMatch.PrefixMatchSPOCMapper.class);
			j5.waitForCompletion(true);
			
			doBulkLoad(bulkLoad, spocPath, currentTable);
			if (onlyTriples == false){//the table with prefix context
				Path cspoPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.CSPO]);
				Path cpsoPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.CPSO]);
				Path ocspPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.OCSP]);
				
				//CSPO----------------------
				Job j7 = createPrefixMatchJob(con, convertedTripletsPath, cspoPath, HBPrefixMatchSchema.CSPO, PrefixMatch.PrefixMatchCSPOMapper.class);
				j7.waitForCompletion(true);
				
				doBulkLoad(bulkLoad, cspoPath, currentTable);
				
				//CPSO-------------------------
				Job j8 = createPrefixMatchJob(con, convertedTripletsPath, cpsoPath, HBPrefixMatchSchema.CPSO, PrefixMatch.PrefixMatchCPSOMapper.class);
				j8.waitForCompletion(true);
				
				doBulkLoad(bulkLoad, cpsoPath, currentTable);
				
				//OCSP--------------------------			
				Job j9 = createPrefixMatchJob(con, convertedTripletsPath, ocspPath, HBPrefixMatchSchema.OCSP, PrefixMatch.PrefixMatchOCSPMapper.class);
				j9.waitForCompletion(true);
				
				doBulkLoad(bulkLoad, ocspPath, currentTable);
void main(String[] args)
@@ -14,9 +14,12 @@
 import nl.vu.datalayer.hbase.id.BaseId;
 import nl.vu.datalayer.hbase.id.DataPair;
 import nl.vu.datalayer.hbase.id.TypedId;
+import nl.vu.datalayer.hbase.id.HBaseValue;
 import nl.vu.datalayer.hbase.schema.HBPrefixMatchSchema;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.HTableInterface;
@@ -25,7 +28,6 @@
 import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat;
 import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
 import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapreduce.Counter;
 import org.apache.hadoop.mapreduce.CounterGroup;
@@ -71,15 +73,211 @@ public class BulkLoad {
 	 */
 	public static double LOAD_BALANCER_FACTOR = 1.2;
 	
-	public static SortedMap<Short, Long> sufixCounters;
+	//public static SortedMap<Short, Long> sufixCounters;
 	
 	public static long totalStringCount, numericalCount, literalCount, bNodeCount;
 	public static int tripleToResourceReduceTasks;
 	
 	public static String schemaSuffix = "";
+
+	private static Path input;
+
+	private static int inputEstimateSize;
+
+	private static String outputPath;
+
+	private static boolean onlyTriples;
+
+	private static NativeJavaConnection con;
 	
-	public static Job createTripleToResourceJob(Path input, Path output, int inputSizeEstimate) throws IOException{
+	/*public static void main(String[] args) {
+		try {
+		Path input = new Path(args[0]);
+		String outputPath = args[2];
+		Path resourceIds = new Path(outputPath+TripleToResource.RESOURCE_IDS_DIR);
+		Path convertedTripletsPath = new Path(outputPath+ResourceToTriple.TEMP_TRIPLETS_DIR);
+		
+		Path string2IdInput = new Path(outputPath+TripleToResource.ID2STRING_DIR);
+		Path string2IdOutput = new Path(outputPath+StringIdAssoc.STRING2ID_DIR);
+		
+		HBaseConnection con = new HBaseConnection(HBaseConnection.JAVA_API);
+		HBPrefixMatchSchema.createCounterTable(con.getAdmin());
+		
+		long start = System.currentTimeMillis();
+		
+		Job j1 = createTripleToResourceJob(input, resourceIds, Integer.parseInt(args[1]));
+		j1.getConfiguration().set("outputPath", outputPath);
+		j1.waitForCompletion(true);
+		
+		long firstJob = System.currentTimeMillis()-start;
+		System.out.println("First pass finished in "+firstJob+" ms");
+		
+		//retrieve counter values and build a SortedMap
+		retrieveCounters(j1);
+		long startPartition = HBPrefixMatchSchema.updateLastCounter(tripleToResourceReduceTasks, con.getConfiguration())+1;
+		}
+		catch (Exception e) {
+			e.printStackTrace();
+		}
+	}*/
+	
+	/**
+	 * @param args
+	 */
+	public static void main(String[] args) {
+		try {
+			if (args.length != 5){
+				System.out.println("Usage: bulkLoad <inputPath> <inputSizeEstimate in MB> <outputPath> <schemaSuffix> <onlyTriples(true/false)>");
+				return;
+			}		
+			initializeLocalVariables(args); 
+			
+			Path convertedTripletsPath = new Path(outputPath+ResourceToTriple.TEMP_TRIPLETS_DIR);
+			Path idStringAssocInput = new Path(outputPath+"/"+TripleToResource.ID2STRING_DIR);
+			
+			con = (NativeJavaConnection)HBaseConnection.create(HBaseConnection.NATIVE_JAVA);
+			HBPrefixMatchSchema prefMatchSchema = new HBPrefixMatchSchema(con, schemaSuffix);
+			prefMatchSchema.createCounterTable(con.getAdmin());
+			
+			long startPartition = runIdGenerationJobs(convertedTripletsPath, idStringAssocInput);
+		
+			//create all tables containing PrefixMatch schema ----------------------
+			System.out.println(totalStringCount+" : "+numericalCount);
+					
+			prefMatchSchema.setTableSplitInfo(totalStringCount, numericalCount, 
+					tripleToResourceReduceTasks, startPartition, onlyTriples);
+			prefMatchSchema.create();
+			
+			LoadIncrementalHFiles bulkLoad = new LoadIncrementalHFiles(con.getConfiguration());
+			
+			bulkLoadIdStringMappingTables(idStringAssocInput, bulkLoad);		
+			bulkLoadQuadTables(convertedTripletsPath, bulkLoad);
+			
+			con.close();		
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+		catch(Exception e){
+			e.printStackTrace();
+		}
+	}
+
+	private static void bulkLoadQuadTables(Path convertedTripletsPath, LoadIncrementalHFiles bulkLoad) throws Exception, IOException, InterruptedException, ClassNotFoundException {
+		Path spocPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.SPOC]);
+		Path pocsPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.POCS]);
+		Path ospcPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.OSPC]);
+		
+		//SPOC--------------------
+		Job j5 = createPrefixMatchJob(con, convertedTripletsPath, spocPath, HBPrefixMatchSchema.SPOC, PrefixMatch.PrefixMatchSPOCMapper.class);
+		j5.waitForCompletion(true);
+		
+		doBulkLoad(bulkLoad, spocPath, currentTable, con);
+
+		if (onlyTriples == false){
+			bulkLoadQuadOnlyTables(convertedTripletsPath, bulkLoad);
+		}
+		
+		//OSPC---------------------------
+		Job j10 = createPrefixMatchJob(con, convertedTripletsPath, ospcPath, HBPrefixMatchSchema.OSPC, PrefixMatch.PrefixMatchOSPCMapper.class);
+		j10.waitForCompletion(true);
+		
+		doBulkLoad(bulkLoad, ospcPath, currentTable, con);
+		
+		//POCS---------------------
+		Job j6 = createPrefixMatchJob(con, convertedTripletsPath, pocsPath, HBPrefixMatchSchema.POCS, PrefixMatch.PrefixMatchPOCSMapper.class);
+		j6.waitForCompletion(true);
+		
+		doBulkLoad(bulkLoad, pocsPath, currentTable, con);
+	}
+
+	private static void bulkLoadQuadOnlyTables(Path convertedTripletsPath, LoadIncrementalHFiles bulkLoad) throws Exception, IOException, InterruptedException, ClassNotFoundException {
+		Path cspoPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.CSPO]);
+		Path cpsoPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.CPSO]);
+		Path ocspPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.OCSP]);
+		
+		//CSPO----------------------
+		Job j7 = createPrefixMatchJob(con, convertedTripletsPath, cspoPath, HBPrefixMatchSchema.CSPO, PrefixMatch.PrefixMatchCSPOMapper.class);
+		j7.waitForCompletion(true);
+		
+		doBulkLoad(bulkLoad, cspoPath, currentTable, con);
+		
+		//CPSO-------------------------
+		Job j8 = createPrefixMatchJob(con, convertedTripletsPath, cpsoPath, HBPrefixMatchSchema.CPSO, PrefixMatch.PrefixMatchCPSOMapper.class);
+		j8.waitForCompletion(true);
+		
+		doBulkLoad(bulkLoad, cpsoPath, currentTable, con);
+		
+		//OCSP--------------------------			
+		Job j9 = createPrefixMatchJob(con, convertedTripletsPath, ocspPath, HBPrefixMatchSchema.OCSP, PrefixMatch.PrefixMatchOCSPMapper.class);
+		j9.waitForCompletion(true);
+		
+		doBulkLoad(bulkLoad, ocspPath, currentTable, con);
+	}
+
+	final private static void bulkLoadIdStringMappingTables(Path idStringAssocInput, LoadIncrementalHFiles bulkLoad) throws Exception, IOException, InterruptedException, ClassNotFoundException {
+		Path string2IdOutput = new Path(outputPath+StringIdAssoc.STRING2ID_DIR);
+		Path id2StringOutput = new Path(outputPath+StringIdAssoc.ID2STRING_DIR);
+		
+		//String2Id PASS----------------------------------------------		
+		Job j3 = createString2IdJob(con, idStringAssocInput, string2IdOutput);
+		j3.waitForCompletion(true);
+		
+		//string2Id = con.getTable(HBPrefixMatchSchema.STRING2ID+schemaSuffix);
+		doBulkLoad(bulkLoad, string2IdOutput, string2Id, con);
+		
+		System.out.println("Finished bulk load for String2Id table");//=====================//=============
+		
+		Job j4 = createId2StringJob(con, idStringAssocInput, id2StringOutput);
+		j4.waitForCompletion(true);
+		
+		doBulkLoad(bulkLoad, id2StringOutput, id2String, con);		
+		System.out.println("Finished bulk load for Id2String table");//=====================//==================================
+	}
+
+	final private static long runIdGenerationJobs(Path convertedTripletsPath, Path idStringAssocInput) throws IOException, InterruptedException, ClassNotFoundException {		
+		Path resourceIds = new Path(outputPath+"/"+TripleToResource.RESOURCE_IDS_DIR);
+		
+		runTripleToResourceJob(idStringAssocInput, resourceIds);
+		long startPartition = HBPrefixMatchSchema.updateLastCounter(tripleToResourceReduceTasks, con.getConfiguration(), schemaSuffix)+1;
+		//long startPartition = HBPrefixMatchSchema.getLastCounter(con.getConfiguration())+1;
+		
+		//SECOND PASS-----------------------------------------------	
+		Job j2 = createResourceToTripleJob(resourceIds, convertedTripletsPath);	
+		j2.waitForCompletion(true);
+		System.out.println("Second pass finished");
+		
+		//read counters from file
+		//buildCountersFromFile();
+		
+		return startPartition;
+	}
+
+	private static void runTripleToResourceJob(Path idStringAssocInput, Path resourceIds) throws IOException, InterruptedException, ClassNotFoundException {
+		long start = System.currentTimeMillis();
+		Job j1 = createTripleToResourceJob(input, resourceIds, inputEstimateSize);
+		j1.getConfiguration().set("outputPath", outputPath);
+		j1.getConfiguration().set("schemaSuffix", schemaSuffix);
+		j1.waitForCompletion(true);
+		
+		//move side effect files out of TripleToResource output directory
+		moveIdStringAssocDirectory(resourceIds, idStringAssocInput);
+		
+		long firstJob = System.currentTimeMillis()-start;
+		System.out.println("First pass finished in "+firstJob+" ms");
 		
+		retrieveTripleToResourceCounters(j1);
+	}
+
+	private static void initializeLocalVariables(String[] args) {
+		input = new Path(args[0]);
+		inputEstimateSize = Integer.parseInt(args[1]);
+		outputPath = args[2];
+		schemaSuffix = args[3];
+		onlyTriples = Boolean.parseBoolean(args[4]);
+	}
+
+
+	public static Job createTripleToResourceJob(Path input, Path output, int inputSizeEstimate) throws IOException{
 		Configuration conf = new Configuration();
 		conf.setInt("mapred.job.reuse.jvm.num.tasks", -1);
 		
@@ -97,7 +295,8 @@ public static Job createTripleToResourceJob(Path input, Path output, int inputSi
 		long numInputBytes = (long)(inputSizeEstimate*Math.pow(10.0, 6.0));
 		long numQuads = numInputBytes/QUAD_AVERAGE_SIZE;
 		long totalNumberOfElements = numQuads*ELEMENTS_PER_QUAD;		
-		tripleToResourceReduceTasks = (int)(Math.ceil((double)totalNumberOfElements/Math.pow(2.0, 24.0)) * LOAD_BALANCER_FACTOR);
+		//tripleToResourceReduceTasks = (int)(Math.ceil((double)totalNumberOfElements/Math.pow(2.0, 24.0)) * LOAD_BALANCER_FACTOR);
+		tripleToResourceReduceTasks = 2;//TODO
 		System.out.println("Number of reduce tasks: "+tripleToResourceReduceTasks);
 		
 		j.setJarByClass(BulkLoad.class);
@@ -107,7 +306,7 @@ public static Job createTripleToResourceJob(Path input, Path output, int inputSi
 		j.setOutputKeyClass(TypedId.class);
 		j.setOutputValueClass(DataPair.class);
 		
-		j.setMapOutputKeyClass(Text.class);
+		j.setMapOutputKeyClass(HBaseValue.class);
 		j.setMapOutputValueClass(DataPair.class);
 
 		j.setInputFormatClass(TextInputFormat.class);
@@ -219,8 +418,8 @@ public static Job createPrefixMatchJob(HBaseConnection con, Path input, Path out
 		return j;
 	}
 	
-	public static void retrieveCounters(Job j1) throws IOException{
-		sufixCounters = new TreeMap<Short, Long>();
+	public static void retrieveTripleToResourceCounters(Job j1) throws IOException{
+		//sufixCounters = new TreeMap<Short, Long>();
 		
 		Counters counters = j1.getCounters();
 		CounterGroup numGroup = counters.getGroup(TripleToResource.TripleToResourceReducer.NUMERICAL_GROUP);
@@ -232,7 +431,7 @@ public static void retrieveCounters(Job j1) throws IOException{
 		bNodeCount = elemsGroup.findCounter("Blanks").getValue();
 		
 		//build histogram
-		CounterGroup histogramGroup = counters.getGroup(TripleToResource.TripleToResourceReducer.HISTOGRAM_GROUP);
+		/*CounterGroup histogramGroup = counters.getGroup(TripleToResource.TripleToResourceReducer.HISTOGRAM_GROUP);
 		Iterator<Counter> it = histogramGroup.iterator();
 		while (it.hasNext()){
 			Counter c = it.next();
@@ -242,29 +441,29 @@ public static void retrieveCounters(Job j1) throws IOException{
 			long half = c.getValue()/2;
 			sufixCounters.put(b, half);
 			sufixCounters.put((short)(b | 0x01), c.getValue()-half);
-		}		
+		}*/		
 		
 		//save counter values to file
 		FileWriter file = new FileWriter("Counters");
 		file.write(Long.toString(totalStringCount)+"\n");
 		file.write(Long.toString(numericalCount)+"\n");
 		file.write(Integer.toString(tripleToResourceReduceTasks)+"\n");
 		
-		for (Map.Entry<Short, Long> entry : sufixCounters.entrySet()) {
+		/*for (Map.Entry<Short, Long> entry : sufixCounters.entrySet()) {
 			file.write(entry.getKey()+"@"+entry.getValue()+"\n");
-		}
+		}*/
 		file.close();
 	}
 	
 	public static void buildCountersFromFile() throws IOException{
-		sufixCounters = new TreeMap();
+		//sufixCounters = new TreeMap();
 		FileReader file2 = new FileReader("Counters");
 		BufferedReader reader = new BufferedReader(file2);
 		totalStringCount = Long.parseLong(reader.readLine());
 		numericalCount = Long.parseLong(reader.readLine());
 		tripleToResourceReduceTasks = Integer.parseInt(reader.readLine());
 		
-		String line;
+		/*String line;
 		long totalCounter = 0;
 		while ((line=reader.readLine()) != null){
 			String []tokens  = line.split("@");
@@ -274,186 +473,40 @@ public static void buildCountersFromFile() throws IOException{
 			sufixCounters.put(s, current);
 			totalCounter += current;
 		}
-		System.out.println(totalCounter);
+		System.out.println(totalCounter);*/
 	}
 
-	private static void doBulkLoad(LoadIncrementalHFiles bulkLoad, Path dir, HTableInterface table) throws InterruptedException{
+	private static void doBulkLoad(LoadIncrementalHFiles bulkLoad, Path dir, HTableInterface table, NativeJavaConnection con) throws InterruptedException{
+		boolean timedOut = false;
 		try{
 			long start = System.currentTimeMillis();
 			bulkLoad.doBulkLoad(dir, (HTable)table);
 			long bulkTime = System.currentTimeMillis()-start;
 			System.out.println(table.getTableDescriptor().getNameAsString()+" bulkLoad time: "+bulkTime);
+				
 		}
 		catch(java.io.IOException e){
-			System.out.println("Bulk load taking longer than usual: sleeping for 5 minutes before proceeding to the next table..");
+			System.out.println("Bulk load taking longer than usual: sleeping for 10 minutes before proceeding to the next table..");
 			Thread.sleep(600000);
+			timedOut = true;
 		}
-	}
-	
-	/*public static void main(String[] args) {
-		try {
-		Path input = new Path(args[0]);
-		String outputPath = args[2];
-		Path resourceIds = new Path(outputPath+TripleToResource.RESOURCE_IDS_DIR);
-		Path convertedTripletsPath = new Path(outputPath+ResourceToTriple.TEMP_TRIPLETS_DIR);
-		
-		Path string2IdInput = new Path(outputPath+TripleToResource.ID2STRING_DIR);
-		Path string2IdOutput = new Path(outputPath+StringIdAssoc.STRING2ID_DIR);
-		
-		HBaseConnection con = new HBaseConnection(HBaseConnection.JAVA_API);
-		HBPrefixMatchSchema.createCounterTable(con.getAdmin());
-		
-		long start = System.currentTimeMillis();
-		
-		Job j1 = createTripleToResourceJob(input, resourceIds, Integer.parseInt(args[1]));
-		j1.getConfiguration().set("outputPath", outputPath);
-		j1.waitForCompletion(true);
 		
-		long firstJob = System.currentTimeMillis()-start;
-		System.out.println("First pass finished in "+firstJob+" ms");
-		
-		//retrieve counter values and build a SortedMap
-		retrieveCounters(j1);
-		long startPartition = HBPrefixMatchSchema.updateLastCounter(tripleToResourceReduceTasks, con.getConfiguration())+1;
-		}
-		catch (Exception e) {
-			e.printStackTrace();
-		}
-	}*/
-	
-	/**
-	 * @param args
-	 */
-	public static void main(String[] args) {
-		try {
-			if (args.length != 5){
-				System.out.println("Usage: bulkLoad <inputPath> <inputSizeEstimate in MB> <outputPath> <schemaSuffix> <onlyTriples(true/false)>");
-				return;
+		if (timedOut){//the BulkLoad took too long - which means the cluster is unbalanced
+			try {
+				con.getAdmin().split(table.getTableName());
+				Thread.sleep(300000);//TODO find a better method to do these splits
+			} catch (IOException e) {
+				e.printStackTrace();
 			}
-			
-			Path input = new Path(args[0]);
-			String outputPath = args[2];
-			schemaSuffix = args[3];
-			boolean onlyTriples = Boolean.parseBoolean(args[4]); 
-			
-			Path resourceIds = new Path(outputPath+TripleToResource.RESOURCE_IDS_DIR);
-			Path convertedTripletsPath = new Path(outputPath+ResourceToTriple.TEMP_TRIPLETS_DIR);
-			
-			Path string2IdInput = new Path(outputPath+TripleToResource.ID2STRING_DIR);
-			Path string2IdOutput = new Path(outputPath+StringIdAssoc.STRING2ID_DIR);
-			
-			NativeJavaConnection con = (NativeJavaConnection)HBaseConnection.create(HBaseConnection.NATIVE_JAVA);
-			HBPrefixMatchSchema prefMatchSchema = new HBPrefixMatchSchema(con, schemaSuffix);
-			prefMatchSchema.createCounterTable(con.getAdmin());
-			
-			long start = System.currentTimeMillis();
-			
-			Job j1 = createTripleToResourceJob(input, resourceIds, Integer.parseInt(args[1]));//TODO
-			j1.getConfiguration().set("outputPath", outputPath);
-			j1.getConfiguration().set("schemaSuffix", schemaSuffix);
-			j1.waitForCompletion(true);
-			
-			long firstJob = System.currentTimeMillis()-start;
-			System.out.println("First pass finished in "+firstJob+" ms");
-			
-			//retrieve counter values and build a SortedMap
-			retrieveCounters(j1);//TODO
-			long startPartition = HBPrefixMatchSchema.updateLastCounter(tripleToResourceReduceTasks, con.getConfiguration(), schemaSuffix)+1;//TODO
-			//long startPartition = HBPrefixMatchSchema.getLastCounter(con.getConfiguration())+1;
-			
-			
-			//SECOND PASS-----------------------------------------------	
-			
-			Job j2 = createResourceToTripleJob(resourceIds, convertedTripletsPath);	
-			j2.waitForCompletion(true);
-			
-			System.out.println("Second pass finished");
-			
-			//read counters from file
-			//buildCountersFromFile();
+		}
 		
-			//create all tables containing PrefixMatch schema ----------------------
-			System.out.println(totalStringCount+" : "+numericalCount);
-					
-			prefMatchSchema.setTableSplitInfo(totalStringCount, numericalCount, 
-					tripleToResourceReduceTasks, startPartition, sufixCounters, 
-					onlyTriples);
-			prefMatchSchema.create();
-			
-			LoadIncrementalHFiles bulkLoad = new LoadIncrementalHFiles(con.getConfiguration());
-			
-			//String2Id PASS----------------------------------------------
-					
-			Job j3 = createString2IdJob(con, string2IdInput, string2IdOutput);
-			j3.waitForCompletion(true);
-			
-			//string2Id = con.getTable(HBPrefixMatchSchema.STRING2ID);
-			doBulkLoad(bulkLoad, string2IdOutput, string2Id);
-			
-			System.out.println("Finished bulk load for String2Id table");//=====================//=============
-			
-			Path id2StringOutput = new Path(outputPath+StringIdAssoc.ID2STRING_DIR);
-			Job j4 = createId2StringJob(con, string2IdInput, id2StringOutput);
-			j4.waitForCompletion(true);
-			
-			doBulkLoad(bulkLoad, id2StringOutput, id2String);
-			
-			System.out.println("Finished bulk load for Id2String table");//=====================//==================================
-			
-			Path spocPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.SPOC]);
-			Path pocsPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.POCS]);
-			Path ospcPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.OSPC]);
-			
-			//SPOC--------------------
-			Job j5 = createPrefixMatchJob(con, convertedTripletsPath, spocPath, HBPrefixMatchSchema.SPOC, PrefixMatch.PrefixMatchSPOCMapper.class);
-			j5.waitForCompletion(true);
-			
-			doBulkLoad(bulkLoad, spocPath, currentTable);
+	}
 	
-			if (onlyTriples == false){//the table with prefix context
-				Path cspoPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.CSPO]);
-				Path cpsoPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.CPSO]);
-				Path ocspPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.OCSP]);
-				
-				//CSPO----------------------
-				Job j7 = createPrefixMatchJob(con, convertedTripletsPath, cspoPath, HBPrefixMatchSchema.CSPO, PrefixMatch.PrefixMatchCSPOMapper.class);
-				j7.waitForCompletion(true);
-				
-				doBulkLoad(bulkLoad, cspoPath, currentTable);
-				
-				//CPSO-------------------------
-				Job j8 = createPrefixMatchJob(con, convertedTripletsPath, cpsoPath, HBPrefixMatchSchema.CPSO, PrefixMatch.PrefixMatchCPSOMapper.class);
-				j8.waitForCompletion(true);
-				
-				doBulkLoad(bulkLoad, cpsoPath, currentTable);
-				
-				//OCSP--------------------------			
-				Job j9 = createPrefixMatchJob(con, convertedTripletsPath, ocspPath, HBPrefixMatchSchema.OCSP, PrefixMatch.PrefixMatchOCSPMapper.class);
-				j9.waitForCompletion(true);
-				
-				doBulkLoad(bulkLoad, ocspPath, currentTable);
-			}
-			
-			//OSPC---------------------------
-			Job j10 = createPrefixMatchJob(con, convertedTripletsPath, ospcPath, HBPrefixMatchSchema.OSPC, PrefixMatch.PrefixMatchOSPCMapper.class);
-			j10.waitForCompletion(true);
-			
-			doBulkLoad(bulkLoad, ospcPath, currentTable);
-			
-			//POCS---------------------
-			Job j6 = createPrefixMatchJob(con, convertedTripletsPath, pocsPath, HBPrefixMatchSchema.POCS, PrefixMatch.PrefixMatchPOCSMapper.class);
-			j6.waitForCompletion(true);
-			
-			doBulkLoad(bulkLoad, pocsPath, currentTable);
-			
-			con.close();
-			
-		} catch (IOException e) {
-			e.printStackTrace();
-		}
-		catch(Exception e){
-			e.printStackTrace();
-		}
+	public static void moveIdStringAssocDirectory(Path resourceIds, Path id2StringInput) throws IOException{
+		Configuration conf = new Configuration();
+		FileSystem fs = FileSystem.get(conf);
+		Path source = new Path(resourceIds, TripleToResource.ID2STRING_DIR);
+		FileUtil.copy(fs, source, fs, id2StringInput, true, false, conf);
 	}
-
+	
 }
https://github.com/pgroth/hbase-rdf/raw/cbf0cac3f03c8dffbc415eab1421234dfc3ead24/src%2Fmain%2Fjava%2Fnl%2Fvu%2Fdatalayer%2Fhbase%2Fbulkload%2FBulkLoad.java
5b176fe9d63525411ece6cfa8457befe0c4a5648
modified
https://github.com/pgroth/hbase-rdf/blob/cbf0cac3f03c8dffbc415eab1421234dfc3ead24/src%2Fmain%2Fjava%2Fnl%2Fvu%2Fdatalayer%2Fhbase%2Fbulkload%2FStringIdAssoc.java
https://api.github.com/repos/pgroth/hbase-rdf/contents/src%2Fmain%2Fjava%2Fnl%2Fvu%2Fdatalayer%2Fhbase%2Fbulkload%2FStringIdAssoc.java?ref=cbf0cac3f03c8dffbc415eab1421234dfc3ead24
src/main/java/nl/vu/datalayer/hbase/bulkload/StringIdAssoc.java
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.UnsupportedEncodingException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import nl.vu.datalayer.hbase.id.HBaseValue;
import org.apache.hadoop.mapreduce.Mapper.Context;
			org.apache.hadoop.mapreduce.Mapper<ImmutableBytesWritable, HBaseValue, ImmutableBytesWritable, Put>
		protected void map(ImmutableBytesWritable key, HBaseValue value, Context context) throws IOException, InterruptedException {
			ByteArrayOutputStream out = new ByteArrayOutputStream();
			value.write(new DataOutputStream(out));
			//byte []valueBytes = new byte [value.getLength()];
			//System.arraycopy(value.getBytes(), 0, valueBytes, 0, value.getLength());
			
			outVal.add(HBPrefixMatchSchema.COLUMN_FAMILY, HBPrefixMatchSchema.COLUMN_NAME, out.toByteArray());
	public static byte []reverseBytes(byte []sourceBytes){
		int lastIndex = sourceBytes.length-1;
			org.apache.hadoop.mapreduce.Mapper<ImmutableBytesWritable, HBaseValue, ImmutableBytesWritable, Put> 
		private MessageDigest mDigest;
		
		protected void setup(Context context) throws IOException, InterruptedException {
			try {
				mDigest = MessageDigest.getInstance("MD5");
			} catch (NoSuchAlgorithmException e) {
				e.printStackTrace();
			}
		}

		@Override
		protected void map(ImmutableBytesWritable inKey, HBaseValue inValue, Context context) 
			byte []md5HashBytes = mDigest.digest(inValue.toString().getBytes("UTF-8"));
			ImmutableBytesWritable outKey = new ImmutableBytesWritable(md5HashBytes);				
			Put outVal = new Put(md5HashBytes);
			org.apache.hadoop.mapreduce.Mapper<ImmutableBytesWritable, Text, ImmutableBytesWritable, Put>
		protected void map(ImmutableBytesWritable key, Text value, Context context) throws IOException, InterruptedException {
			byte []valueBytes = new byte [value.getLength()];
			System.arraycopy(value.getBytes(), 0, valueBytes, 0, value.getLength());
			outVal.add(HBPrefixMatchSchema.COLUMN_FAMILY, HBPrefixMatchSchema.COLUMN_NAME, valueBytes);
	public static byte []reverseBytes(byte []sourceBytes, int sourceLength){
		int lastIndex;
		if (sourceBytes[sourceLength-1] == '\"')//eliminate last " for literals in order to have a good dispersion
			lastIndex = sourceLength-2;
		else
			lastIndex = sourceLength-1;
			org.apache.hadoop.mapreduce.Mapper<ImmutableBytesWritable, Text, ImmutableBytesWritable, Put> 
		protected void map(ImmutableBytesWritable inKey, Text inValue, Context context) 
			byte []inValueBytes;
			byte []sourceBytes = inValue.getBytes();
			int sourceLength = inValue.getLength();
			if (sourceLength > HConstants.MAX_ROW_LENGTH){//if the key is too long use the hash
				String s = Text.decode(sourceBytes, 0, sourceLength);
				long hash = hashFunction(s);
				inValueBytes = new byte[9];
				//the first byte is left 00, because it is not part of RDF characters
				//it acts as an escape in case all the bytes in the hash are valid RDF characters
				Bytes.putLong(inValueBytes, 1, hash);
			}
			else{//reverse the value so that we have a better dispersion for links which have the same prefix		
				inValueBytes = reverseBytes(sourceBytes, sourceLength);
			}		
			ImmutableBytesWritable outKey = new ImmutableBytesWritable(inValueBytes);				
			Put outVal = new Put(inValueBytes);
@@ -1,34 +1,44 @@
 package nl.vu.datalayer.hbase.bulkload;
 
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
 import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
 
+import nl.vu.datalayer.hbase.id.HBaseValue;
 import nl.vu.datalayer.hbase.schema.HBPrefixMatchSchema;
 
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Mapper.Context;
 
 public class StringIdAssoc {
 	
 	public static final String STRING2ID_DIR = "/string2Id";
 	public static final String ID2STRING_DIR = "/id2String";
 	
 	public static class Id2StringMapper extends 
-			org.apache.hadoop.mapreduce.Mapper<ImmutableBytesWritable, Text, ImmutableBytesWritable, Put>
+			org.apache.hadoop.mapreduce.Mapper<ImmutableBytesWritable, HBaseValue, ImmutableBytesWritable, Put>
 	{
 		@Override
-		protected void map(ImmutableBytesWritable key, Text value, Context context) throws IOException, InterruptedException {
+		protected void map(ImmutableBytesWritable key, HBaseValue value, Context context) throws IOException, InterruptedException {
 			
 			byte []keyBytes = new byte [key.getLength()];
 			System.arraycopy(key.get(), key.getOffset(), keyBytes, 0, key.getLength());
 			
 			Put outVal = new Put(keyBytes);
 			
-			byte []valueBytes = new byte [value.getLength()];
-			System.arraycopy(value.getBytes(), 0, valueBytes, 0, value.getLength());
-			outVal.add(HBPrefixMatchSchema.COLUMN_FAMILY, HBPrefixMatchSchema.COLUMN_NAME, valueBytes);
+			ByteArrayOutputStream out = new ByteArrayOutputStream();
+			value.write(new DataOutputStream(out));
+			//byte []valueBytes = new byte [value.getLength()];
+			//System.arraycopy(value.getBytes(), 0, valueBytes, 0, value.getLength());
+			
+			outVal.add(HBPrefixMatchSchema.COLUMN_FAMILY, HBPrefixMatchSchema.COLUMN_NAME, out.toByteArray());
 			
 			context.write(key, outVal);
 		}
@@ -49,12 +59,8 @@ private static long hashFunction(String key){
 		return hash;
 	}
 	
-	public static byte []reverseBytes(byte []sourceBytes, int sourceLength){
-		int lastIndex;
-		if (sourceBytes[sourceLength-1] == '\"')//eliminate last " for literals in order to have a good dispersion
-			lastIndex = sourceLength-2;
-		else
-			lastIndex = sourceLength-1;
+	public static byte []reverseBytes(byte []sourceBytes){
+		int lastIndex = sourceBytes.length-1;
 		
 		byte []outValueBytes = new byte[lastIndex+1];
 		for (int i = 0; i < outValueBytes.length; i++) {
@@ -64,29 +70,27 @@ private static long hashFunction(String key){
 	}
 	
 	public static class String2IdMapper extends 
-			org.apache.hadoop.mapreduce.Mapper<ImmutableBytesWritable, Text, ImmutableBytesWritable, Put> 
+			org.apache.hadoop.mapreduce.Mapper<ImmutableBytesWritable, HBaseValue, ImmutableBytesWritable, Put> 
 	{
+		private MessageDigest mDigest;
+		
 		@Override
-		protected void map(ImmutableBytesWritable inKey, Text inValue, Context context) 
+		protected void setup(Context context) throws IOException, InterruptedException {
+			try {
+				mDigest = MessageDigest.getInstance("MD5");
+			} catch (NoSuchAlgorithmException e) {
+				e.printStackTrace();
+			}
+		}
+
+		@Override
+		protected void map(ImmutableBytesWritable inKey, HBaseValue inValue, Context context) 
 										throws IOException, InterruptedException {
-			byte []inValueBytes;
-			byte []sourceBytes = inValue.getBytes();
-			int sourceLength = inValue.getLength();
 			
-			if (sourceLength > HConstants.MAX_ROW_LENGTH){//if the key is too long use the hash
-				String s = Text.decode(sourceBytes, 0, sourceLength);
-				long hash = hashFunction(s);
-				inValueBytes = new byte[9];
-				//the first byte is left 00, because it is not part of RDF characters
-				//it acts as an escape in case all the bytes in the hash are valid RDF characters
-				Bytes.putLong(inValueBytes, 1, hash);
-			}
-			else{//reverse the value so that we have a better dispersion for links which have the same prefix		
-				inValueBytes = reverseBytes(sourceBytes, sourceLength);
-			}		
-			ImmutableBytesWritable outKey = new ImmutableBytesWritable(inValueBytes);				
+			byte []md5HashBytes = mDigest.digest(inValue.toString().getBytes("UTF-8"));
+			ImmutableBytesWritable outKey = new ImmutableBytesWritable(md5HashBytes);				
 
-			Put outVal = new Put(inValueBytes);
+			Put outVal = new Put(md5HashBytes);
 			byte []inKeyBytes = new byte [inKey.getLength()];
 			System.arraycopy(inKey.get(), inKey.getOffset(), inKeyBytes, 0, inKey.getLength());
 			outVal.add(HBPrefixMatchSchema.COLUMN_FAMILY, HBPrefixMatchSchema.COLUMN_NAME, inKeyBytes);
https://github.com/pgroth/hbase-rdf/raw/cbf0cac3f03c8dffbc415eab1421234dfc3ead24/src%2Fmain%2Fjava%2Fnl%2Fvu%2Fdatalayer%2Fhbase%2Fbulkload%2FStringIdAssoc.java
42a39479d3e024a7e524d737a8b0d049d0b291f0
modified
https://github.com/pgroth/hbase-rdf/blob/cbf0cac3f03c8dffbc415eab1421234dfc3ead24/src%2Fmain%2Fjava%2Fnl%2Fvu%2Fdatalayer%2Fhbase%2Fbulkload%2FTripleToResource.java
https://api.github.com/repos/pgroth/hbase-rdf/contents/src%2Fmain%2Fjava%2Fnl%2Fvu%2Fdatalayer%2Fhbase%2Fbulkload%2FTripleToResource.java?ref=cbf0cac3f03c8dffbc415eab1421234dfc3ead24
src/main/java/nl/vu/datalayer/hbase/bulkload/TripleToResource.java
import nl.vu.datalayer.hbase.exceptions.NumericalRangeException;
import nl.vu.datalayer.hbase.id.HBaseValue;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.openrdf.model.BNode;
import org.openrdf.model.Value;
import org.openrdf.model.impl.URIImpl;
	public static final String RESOURCE_IDS_DIR = "resourceIds";
	public static final String ID2STRING_DIR = "idStringAssoc";
				org.apache.hadoop.mapreduce.Mapper<LongWritable, Writable, HBaseValue, DataPair>
				Statement s = getNewStatement(value);
				BaseId quadId = new BaseId(partitionId, localRowCount);
				DataPair []outputPairs = createDataPairs(quadId);
				emitOutputPairs(context, s, outputPairs);
				context.getCounter(EXCEPTION_GROUP, "RDFParseExceptions").increment(1);
				context.getCounter(EXCEPTION_GROUP, "RDFHandlerExceptions").increment(1);

		final private void emitOutputPairs(Context context, Statement s, DataPair[] dataPairs) throws IOException, InterruptedException {
			context.write(new HBaseValue(s.getSubject()), dataPairs[DataPair.S]);
			context.write(new HBaseValue(s.getPredicate()), dataPairs[DataPair.P]);
			context.write(new HBaseValue(s.getObject()), dataPairs[DataPair.O]);
			
			Value quadContext;
			if (s.getContext() == null){
				quadContext = new URIImpl(DEFAULT_CONTEXT);
			}
			else{
				quadContext = s.getContext();
			}
			context.write(new HBaseValue(quadContext), dataPairs[DataPair.C]);
		}
		
		final private DataPair []createDataPairs(BaseId id){
			DataPair []outputPair = new DataPair[4];
			for (int i = 0; i < outputPair.length; i++) {
				outputPair[i] = new DataPair(id, (byte)i);
			}
			return outputPair;
		}

		final private Statement getNewStatement(Writable value) throws IOException, RDFParseException, RDFHandlerException {
			InputStream in = new ByteArrayInputStream(value.toString().getBytes());
			parser.parse(in, "");
			if (statements.size() == 0){
				return null;
			}							
			Statement ret = statements.get(statements.size()-1);
			if (statements.size() == 5000){
				statements.clear();
			}
			
			return ret;
		}
	public static class TripleToResourceReducer extends org.apache.hadoop.mapreduce.Reducer<HBaseValue, DataPair, TypedId, DataPair>
			//public static final String HISTOGRAM_GROUP = "HistogramGroup";
	
				Configuration config = context.getConfiguration();
				partitionId = config.getInt("mapred.task.partition", 0);
				schemaSuffix = config.get("schemaSuffix");
				String outputPath = config.get("outputPath");
				fs = FileSystem.get(config);
	
				valFact = new ValueFactoryImpl();
	
				Path sideEffectFile = new Path(FileOutputFormat.getWorkOutputPath(context), 
						ID2STRING_DIR + String.format("/%05d", partitionId));
				id2StringWriter = SequenceFile.createWriter(fs, config, sideEffectFile,
				//new Path(outputPath+ID2STRING_DIR+"/"+context.getTaskAttemptID()),
				//new Path("idStringAssoc"+String.format("/%05d", partitionId)),
						ImmutableBytesWritable.class, HBaseValue.class, CompressionType.RECORD);
				//id2StringWriter.syncFs();
			/*public void updateHistogram(Context context, Value literal){
				byte []bytes = literal.stringValue().getBytes();
				context.getCounter(HISTOGRAM_GROUP, 
						String.format("%02x", (short)bytes[bytes.length-1] & 0xfe)).increment(1);
				
			}*/
			public void reduce(HBaseValue key, Iterable<DataPair> values, Context context) throws IOException, InterruptedException {
				try{
					TypedId newId = generateTypedId(context, key.getUnderlyingValue());				
					saveTypeIdToFile(newId, key, context);
					
					//emit (elementID, <tripleID, position>) elements
					for (DataPair dataPair : values) {
						context.write(newId, dataPair);
				catch(NumericalRangeException e){
					System.err.println("Numerical not in expected range: "+e.getMessage());
					context.getCounter(EXCEPTION_GROUP, "RangeExceptions").increment(1);
					return;
				}
			}

			final private TypedId generateTypedId(Context context, Value elem) throws IOException, NumericalRangeException {
				if (elem instanceof Literal){//Literals				
					return handleLiteral(context, elem);
				}
					//context.getCounter(HISTOGRAM_GROUP, String.format("%02x", (short)elemBytes[elemBytes.length-1] & 0xfe)).increment(1);
					return handleNonLiteral(context, elem);
				}
			}
			
			final private TypedId handleLiteral(Context context, Value elem) throws IOException, NumericalRangeException{
				Counter c = context.getCounter(ELEMENT_TYPE_GROUP, "Literals"); 
				c.increment(1);
				Literal l = (Literal)elem;
				if (l.getDatatype() == null){//the Literals with no datatype are considered Strings
					return new TypedId(partitionId, localRowCount);
					//updateHistogram(context, elem);
				}
				else{//we have a datatype
					TypedId newId = TypedId.createNumerical(l);
					if (newId == null){//non-numerical literal
						newId = new TypedId(partitionId, localRowCount);
						//updateHistogram(context, elem);
					return newId;
			}
			
			final private TypedId handleNonLiteral(Context context, Value elem){
				increaseNonLiteralCounters(elem, context);
				return new TypedId(partitionId, localRowCount);
			}
			
			final private void increaseNonLiteralCounters(Value elem, Context context){
				if (elem instanceof BNode){
					context.getCounter(ELEMENT_TYPE_GROUP, "Blanks").increment(1);
				}
				else{
					context.getCounter(ELEMENT_TYPE_GROUP, "URIs").increment(1);
				}
			}	
			
			final private void saveTypeIdToFile(TypedId newId, HBaseValue value, Context context) throws IOException{
					id2StringWriter.append(new ImmutableBytesWritable(newId.getContent()), value);
					/*if (localRowCount % 1000 == 0){
					}*/
	public static void main(String[] args) {		
		Job j1;
		try {
			//Path p = new Path(new Path("/test1"), new Path(ID2STRING_DIR));
			//System.out.println(p);
		
			if (args.length != 3){
				System.out.println("Usage: TripleToResource <inputPath> <inputSizeEstimate in MB> <outputPath>");
				return;
			}
			
			Path input = new Path(args[0]);
			String outputPath = args[2];
			
			Path resourceIds = new Path(outputPath+TripleToResource.RESOURCE_IDS_DIR);
			
			j1 = BulkLoad.createTripleToResourceJob(input, resourceIds, Integer.parseInt(args[1]));
			
			j1.getConfiguration().set("outputPath", outputPath);
			j1.getConfiguration().set("schemaSuffix", "_TEST");
			j1.waitForCompletion(true);
			
			Configuration conf = new Configuration();
			FileSystem fs = FileSystem.get(conf);
			Path source = new Path(resourceIds, "idStringAssoc");
			Path id2String = new Path(outputPath+"/idStringAssoc");
			FileUtil.copy(fs, source, fs, id2String, true, false, conf);
			
		} catch (NumberFormatException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
		catch (InterruptedException e) {
			e.printStackTrace();
		} catch (ClassNotFoundException e) {
			e.printStackTrace();
		}

	}
Context context  Value literal
src/main/java/nl/vu/datalayer/hbase/bulkload/TripleToResource.java
byte []bytes = literal.stringValue().getBytes();
				context.getCounter(HISTOGRAM_GROUP, 
						String.format("%02x", (short)bytes[bytes.length-1] & 0xfe)).increment(1);
void updateHistogram(Context context, Value literal)
DataPair dataPair : values
src/main/java/nl/vu/datalayer/hbase/bulkload/TripleToResource.java
context.write(newId, dataPair);
				catch(NumericalRangeException e){
					System.err.println("Numerical not in expected range: "+e.getMessage());
					context.getCounter(EXCEPTION_GROUP, "RangeExceptions").increment(1);
					return;
elements
					for (DataPair dataPair : values)
Context context  Value elem
src/main/java/nl/vu/datalayer/hbase/bulkload/TripleToResource.java
increaseNonLiteralCounters(elem, context);
				return new TypedId(partitionId, localRowCount);
TypedId handleNonLiteral(Context context, Value elem)
Value elem  Context context
src/main/java/nl/vu/datalayer/hbase/bulkload/TripleToResource.java
if (elem instanceof BNode){
					context.getCounter(ELEMENT_TYPE_GROUP, "Blanks").increment(1);
void increaseNonLiteralCounters(Value elem, Context context)
String[] args
src/main/java/nl/vu/datalayer/hbase/bulkload/TripleToResource.java
Job j1;
		try {
			//Path p = new Path(new Path("/test1"), new Path(ID2STRING_DIR));
			//System.out.println(p);
		
			if (args.length != 3){
				System.out.println("Usage: TripleToResource <inputPath> <inputSizeEstimate in MB> <outputPath>");
				return;
void main(String[] args)
import nl.vu.datalayer.hbase.id.NumericalRangeException;
import org.apache.hadoop.io.Text;
import org.openrdf.rio.ntriples.NTriplesUtil;
	public static final String RESOURCE_IDS_DIR = "/resourceIds";
	public static final String ID2STRING_DIR = "/idStringAssoc";
				org.apache.hadoop.mapreduce.Mapper<LongWritable, Writable, Text, DataPair>
				InputStream in = new ByteArrayInputStream(value.toString().getBytes());
				parser.parse(in, "");
				if (statements.size() == 0){
					return;
				}				
				
				Statement s = statements.get(statements.size()-1);
				
				//generate triple ID
				BaseId id = new BaseId(partitionId, localRowCount);
				//create output pairs
				DataPair subject = new DataPair(id, DataPair.S);
				DataPair predicate = new DataPair(id, DataPair.P);
				DataPair object = new DataPair(id, DataPair.O);
				DataPair tContext = new DataPair(id, DataPair.C);
				//emit output
				context.write(new Text(s.getSubject().toString()), subject);
				context.write(new Text(s.getPredicate().toString()), predicate);
				context.write(new Text(s.getObject().toString()), object);
				
				String quadContext;
				if (s.getContext() == null)
					quadContext = DEFAULT_CONTEXT;
				else
					quadContext = s.getContext().toString();
				context.write(new Text(quadContext), tContext);
				
				if (statements.size() == 5000){
					statements.clear();
				}
				context.getCounter(EXCEPTION_GROUP, "RDDParseExceptions").increment(1);
				context.getCounter(EXCEPTION_GROUP, "RDDHandlerExceptions").increment(1);
	public static class TripleToResourceReducer extends org.apache.hadoop.mapreduce.Reducer<Text, DataPair, TypedId, DataPair>
			public static final String HISTOGRAM_GROUP = "HistogramGroup";
				try {
					Configuration config = context.getConfiguration();
					partitionId = config.getInt("mapred.task.partition", 0);
					schemaSuffix = config.get("schemaSuffix");
					String outputPath = config.get("outputPath");
					fs = FileSystem.get(config);
					
					valFact = new ValueFactoryImpl();
					
					id2StringWriter = SequenceFile.createWriter(fs, config, 
							new Path(outputPath+ID2STRING_DIR+String.format("/%05d", partitionId)),
							//new Path("idStringAssoc"+String.format("/%05d", partitionId)),
							ImmutableBytesWritable.class, Text.class,
							CompressionType.RECORD);
				} catch (IOException e) {
					e.printStackTrace();
				}
				id2StringWriter.syncFs();
			public void updateHistogram(Context context, String literal){
				if (literal.endsWith("\"")){
					context.getCounter(HISTOGRAM_GROUP, String.format("%02x", (short)literal.getBytes()[literal.getBytes().length-2] & 0xfe)).increment(1);
				}
				else{
					context.getCounter(HISTOGRAM_GROUP, String.format("%02x", (short)literal.getBytes()[literal.getBytes().length-1] & 0xfe)).increment(1);
				}
			}
			public void reduce(Text key, Iterable<DataPair> values, Context context) throws IOException, InterruptedException {
				
				TypedId newId;
				String elem = key.toString();			
				if (elem.startsWith("\"")){//Literals
					Counter c = context.getCounter(ELEMENT_TYPE_GROUP, "Literals"); 
					c.increment(1);
					try{
						Literal l = NTriplesUtil.parseLiteral(elem, valFact);
						if (l.getDatatype() == null){//the Literals with no datatype are considered Strings
							newId = new TypedId(partitionId, localRowCount);
							updateHistogram(context, elem);
						}
						else{//we have a datatype
							newId = TypedId.createNumerical(l);
							if (newId == null){//non-numerical literal
								newId = new TypedId(partitionId, localRowCount);
								updateHistogram(context, elem);
							}
						}
					}
					catch(NumericalRangeException e){
						System.err.println("Numerical not in expected range: "+e.getMessage());
						context.getCounter(EXCEPTION_GROUP, "RangeExceptions").increment(1);
						return;
					}
					catch(IllegalArgumentException e){
						System.err.println("Literal not in proper format: "+e.getMessage());
						context.getCounter(EXCEPTION_GROUP, "LiteralExceptions").increment(1); 
						return;
					context.getCounter(HISTOGRAM_GROUP, String.format("%02x", (short)elem.getBytes()[elem.getBytes().length-1] & 0xfe)).increment(1);
					if (elem.startsWith("_:")){
						context.getCounter(ELEMENT_TYPE_GROUP, "Blanks").increment(1);
					}
					else{
						context.getCounter(ELEMENT_TYPE_GROUP, "URIs").increment(1);
					
					newId = new TypedId(partitionId, localRowCount);
				
				//write the association between IDs and Strings in a SequenceFile
					id2StringWriter.append(new ImmutableBytesWritable(newId.getContent()), key);
					if (localRowCount % 1000 == 0){
					}
				
				//emit (elementID, <tripleID, position>) elements
				for (DataPair dataPair : values) {
					context.write(newId, dataPair);
				}										
					
Context context  String literal
src/main/java/nl/vu/datalayer/hbase/bulkload/TripleToResource.java
if (literal.endsWith("\"")){
					context.getCounter(HISTOGRAM_GROUP, String.format("%02x", (short)literal.getBytes()[literal.getBytes().length-2] & 0xfe)).increment(1);
void updateHistogram(Context context, String literal)
DataPair dataPair : values
src/main/java/nl/vu/datalayer/hbase/bulkload/TripleToResource.java
context.write(newId, dataPair);
elements
				for (DataPair dataPair : values)
@@ -6,43 +6,48 @@
 import java.util.ArrayList;
 
 import net.fortytwo.sesametools.nquads.NQuadsParser;
+import nl.vu.datalayer.hbase.exceptions.NumericalRangeException;
 import nl.vu.datalayer.hbase.id.BaseId;
 import nl.vu.datalayer.hbase.id.DataPair;
-import nl.vu.datalayer.hbase.id.NumericalRangeException;
+import nl.vu.datalayer.hbase.id.HBaseValue;
 import nl.vu.datalayer.hbase.id.TypedId;
 import nl.vu.datalayer.hbase.schema.HBPrefixMatchSchema;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.SequenceFile;
 import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.openrdf.model.BNode;
 import org.openrdf.model.Literal;
 import org.openrdf.model.Statement;
+import org.openrdf.model.Value;
 import org.openrdf.model.ValueFactory;
+import org.openrdf.model.impl.URIImpl;
 import org.openrdf.model.impl.ValueFactoryImpl;
 import org.openrdf.rio.RDFHandlerException;
 import org.openrdf.rio.RDFParseException;
 import org.openrdf.rio.RDFParser;
 import org.openrdf.rio.helpers.StatementCollector;
-import org.openrdf.rio.ntriples.NTriplesUtil;
 
 public class TripleToResource {
 	
-	public static final String RESOURCE_IDS_DIR = "/resourceIds";
-	public static final String ID2STRING_DIR = "/idStringAssoc";
+	public static final String RESOURCE_IDS_DIR = "resourceIds";
+	public static final String ID2STRING_DIR = "idStringAssoc";
 	public static final String DEFAULT_CONTEXT = "http://DEFAULT_CONTEXT";
 	
 	public static final String EXCEPTION_GROUP = "ExceptionGroup";
 	
 	public static class TripleToResourceMapper extends 
-				org.apache.hadoop.mapreduce.Mapper<LongWritable, Writable, Text, DataPair>
+				org.apache.hadoop.mapreduce.Mapper<LongWritable, Writable, HBaseValue, DataPair>
 	{
 		private static RDFParser parser = null;
 		private static StatementCollector collector = null;
@@ -68,49 +73,60 @@ protected void setup(Context context) throws IOException, InterruptedException {
 		@Override
 		public void map(LongWritable key, Writable value, Context context) throws IOException, InterruptedException{
 			try {
-				InputStream in = new ByteArrayInputStream(value.toString().getBytes());
-				parser.parse(in, "");
+				Statement s = getNewStatement(value);
 				
-				if (statements.size() == 0){
-					return;
-				}				
-				
-				Statement s = statements.get(statements.size()-1);
-				
-				//generate triple ID
-				BaseId id = new BaseId(partitionId, localRowCount);
+				BaseId quadId = new BaseId(partitionId, localRowCount);
 				localRowCount++;
 				
-				//create output pairs
-				DataPair subject = new DataPair(id, DataPair.S);
-				DataPair predicate = new DataPair(id, DataPair.P);
-				DataPair object = new DataPair(id, DataPair.O);
-				DataPair tContext = new DataPair(id, DataPair.C);
+				DataPair []outputPairs = createDataPairs(quadId);
 				
-				//emit output
-				context.write(new Text(s.getSubject().toString()), subject);
-				context.write(new Text(s.getPredicate().toString()), predicate);
-				context.write(new Text(s.getObject().toString()), object);
-				
-				String quadContext;
-				if (s.getContext() == null)
-					quadContext = DEFAULT_CONTEXT;
-				else
-					quadContext = s.getContext().toString();
-				context.write(new Text(quadContext), tContext);
-				
-				if (statements.size() == 5000){
-					statements.clear();
-				}
+				emitOutputPairs(context, s, outputPairs);
 				
 			} catch (RDFParseException e) {
 				System.err.println("Unexpected input at line: "+key.get()+" : "+e.getMessage());
-				context.getCounter(EXCEPTION_GROUP, "RDDParseExceptions").increment(1);
+				context.getCounter(EXCEPTION_GROUP, "RDFParseExceptions").increment(1);
 			} catch (RDFHandlerException e) {
 				System.err.println("Line: "+key.get()+"; "+e.getMessage());
-				context.getCounter(EXCEPTION_GROUP, "RDDHandlerExceptions").increment(1);
+				context.getCounter(EXCEPTION_GROUP, "RDFHandlerExceptions").increment(1);
 			} 
 		}
+
+		final private void emitOutputPairs(Context context, Statement s, DataPair[] dataPairs) throws IOException, InterruptedException {
+			context.write(new HBaseValue(s.getSubject()), dataPairs[DataPair.S]);
+			context.write(new HBaseValue(s.getPredicate()), dataPairs[DataPair.P]);
+			context.write(new HBaseValue(s.getObject()), dataPairs[DataPair.O]);
+			
+			Value quadContext;
+			if (s.getContext() == null){
+				quadContext = new URIImpl(DEFAULT_CONTEXT);
+			}
+			else{
+				quadContext = s.getContext();
+			}
+			context.write(new HBaseValue(quadContext), dataPairs[DataPair.C]);
+		}
+		
+		final private DataPair []createDataPairs(BaseId id){
+			DataPair []outputPair = new DataPair[4];
+			for (int i = 0; i < outputPair.length; i++) {
+				outputPair[i] = new DataPair(id, (byte)i);
+			}
+			return outputPair;
+		}
+
+		final private Statement getNewStatement(Writable value) throws IOException, RDFParseException, RDFHandlerException {
+			InputStream in = new ByteArrayInputStream(value.toString().getBytes());
+			parser.parse(in, "");
+			if (statements.size() == 0){
+				return null;
+			}							
+			Statement ret = statements.get(statements.size()-1);
+			if (statements.size() == 5000){
+				statements.clear();
+			}
+			
+			return ret;
+		}
 	
 		@Override
 		protected void cleanup(Context context) throws IOException, InterruptedException {
@@ -119,7 +135,7 @@ protected void cleanup(Context context) throws IOException, InterruptedException
 	}
 	
 	
-	public static class TripleToResourceReducer extends org.apache.hadoop.mapreduce.Reducer<Text, DataPair, TypedId, DataPair>
+	public static class TripleToResourceReducer extends org.apache.hadoop.mapreduce.Reducer<HBaseValue, DataPair, TypedId, DataPair>
 	{
 			private int partitionId;
 			private long localRowCount = 0;
@@ -139,36 +155,34 @@ public static class TripleToResourceReducer extends org.apache.hadoop.mapreduce.
 			 * so 2 byte values are counted with the same counter, and we assume that they are equally distributed
 			 * TODO change in future releases when counter limit can be configured
 			 */
-			public static final String HISTOGRAM_GROUP = "HistogramGroup";
+			//public static final String HISTOGRAM_GROUP = "HistogramGroup";
 			public static final String NUMERICAL_GROUP = "NumericalGroup";		
 			
 			private String schemaSuffix;
 			
 			@Override
 			protected void setup(Context context) throws IOException, InterruptedException {
 				FileSystem fs;
-				try {
-					Configuration config = context.getConfiguration();
-					partitionId = config.getInt("mapred.task.partition", 0);
-					schemaSuffix = config.get("schemaSuffix");
-					String outputPath = config.get("outputPath");
-					fs = FileSystem.get(config);
-					
-					valFact = new ValueFactoryImpl();
-					
-					id2StringWriter = SequenceFile.createWriter(fs, config, 
-							new Path(outputPath+ID2STRING_DIR+String.format("/%05d", partitionId)),
-							//new Path("idStringAssoc"+String.format("/%05d", partitionId)),
-							ImmutableBytesWritable.class, Text.class,
-							CompressionType.RECORD);
-				} catch (IOException e) {
-					e.printStackTrace();
-				}
+	
+				Configuration config = context.getConfiguration();
+				partitionId = config.getInt("mapred.task.partition", 0);
+				schemaSuffix = config.get("schemaSuffix");
+				String outputPath = config.get("outputPath");
+				fs = FileSystem.get(config);
+	
+				valFact = new ValueFactoryImpl();
+	
+				Path sideEffectFile = new Path(FileOutputFormat.getWorkOutputPath(context), 
+						ID2STRING_DIR + String.format("/%05d", partitionId));
+				id2StringWriter = SequenceFile.createWriter(fs, config, sideEffectFile,
+				//new Path(outputPath+ID2STRING_DIR+"/"+context.getTaskAttemptID()),
+				//new Path("idStringAssoc"+String.format("/%05d", partitionId)),
+						ImmutableBytesWritable.class, HBaseValue.class, CompressionType.RECORD);
 			}
 			
 			@Override
 			protected void cleanup(Context context) throws IOException, InterruptedException {
-				id2StringWriter.syncFs();
+				//id2StringWriter.syncFs();
 				id2StringWriter.close();
 				
 				System.out.println("Closed file writer");
@@ -181,80 +195,127 @@ public static void setValueVatory(ValueFactory valFactory){
 				valFact = valFactory;
 			}
 			
-			public void updateHistogram(Context context, String literal){
-				if (literal.endsWith("\"")){
-					context.getCounter(HISTOGRAM_GROUP, String.format("%02x", (short)literal.getBytes()[literal.getBytes().length-2] & 0xfe)).increment(1);
-				}
-				else{
-					context.getCounter(HISTOGRAM_GROUP, String.format("%02x", (short)literal.getBytes()[literal.getBytes().length-1] & 0xfe)).increment(1);
-				}
-			}
+			/*public void updateHistogram(Context context, Value literal){
+				byte []bytes = literal.stringValue().getBytes();
+				context.getCounter(HISTOGRAM_GROUP, 
+						String.format("%02x", (short)bytes[bytes.length-1] & 0xfe)).increment(1);
+				
+			}*/
 
 			@Override
-			public void reduce(Text key, Iterable<DataPair> values, Context context) throws IOException, InterruptedException {
+			public void reduce(HBaseValue key, Iterable<DataPair> values, Context context) throws IOException, InterruptedException {
 				//generate a new id for the triple element represented by the key
-				
-				TypedId newId;
-				String elem = key.toString();			
-				if (elem.startsWith("\"")){//Literals
-					Counter c = context.getCounter(ELEMENT_TYPE_GROUP, "Literals"); 
-					c.increment(1);
-					try{
-						Literal l = NTriplesUtil.parseLiteral(elem, valFact);
-						if (l.getDatatype() == null){//the Literals with no datatype are considered Strings
-							newId = new TypedId(partitionId, localRowCount);
-							updateHistogram(context, elem);
-						}
-						else{//we have a datatype
-							newId = TypedId.createNumerical(l);
-							if (newId == null){//non-numerical literal
-								newId = new TypedId(partitionId, localRowCount);
-								updateHistogram(context, elem);
-							}
-						}
-					}
-					catch(NumericalRangeException e){
-						System.err.println("Numerical not in expected range: "+e.getMessage());
-						context.getCounter(EXCEPTION_GROUP, "RangeExceptions").increment(1);
-						return;
-					}
-					catch(IllegalArgumentException e){
-						System.err.println("Literal not in proper format: "+e.getMessage());
-						context.getCounter(EXCEPTION_GROUP, "LiteralExceptions").increment(1); 
-						return;
+				try{
+					TypedId newId = generateTypedId(context, key.getUnderlyingValue());				
+					saveTypeIdToFile(newId, key, context);
+					
+					//emit (elementID, <tripleID, position>) elements
+					for (DataPair dataPair : values) {
+						context.write(newId, dataPair);
 					}	
 				}
+				catch(NumericalRangeException e){
+					System.err.println("Numerical not in expected range: "+e.getMessage());
+					context.getCounter(EXCEPTION_GROUP, "RangeExceptions").increment(1);
+					return;
+				}
+			}
+
+			final private TypedId generateTypedId(Context context, Value elem) throws IOException, NumericalRangeException {
+				if (elem instanceof Literal){//Literals				
+					return handleLiteral(context, elem);
+				}
 				else{//non-Literals
-					context.getCounter(HISTOGRAM_GROUP, String.format("%02x", (short)elem.getBytes()[elem.getBytes().length-1] & 0xfe)).increment(1);
-					if (elem.startsWith("_:")){
-						context.getCounter(ELEMENT_TYPE_GROUP, "Blanks").increment(1);
-					}
-					else{
-						context.getCounter(ELEMENT_TYPE_GROUP, "URIs").increment(1);
+					//context.getCounter(HISTOGRAM_GROUP, String.format("%02x", (short)elemBytes[elemBytes.length-1] & 0xfe)).increment(1);
+					return handleNonLiteral(context, elem);
+				}
+			}
+			
+			final private TypedId handleLiteral(Context context, Value elem) throws IOException, NumericalRangeException{
+				Counter c = context.getCounter(ELEMENT_TYPE_GROUP, "Literals"); 
+				c.increment(1);
+				Literal l = (Literal)elem;
+				if (l.getDatatype() == null){//the Literals with no datatype are considered Strings
+					return new TypedId(partitionId, localRowCount);
+					//updateHistogram(context, elem);
+				}
+				else{//we have a datatype
+					TypedId newId = TypedId.createNumerical(l);
+					if (newId == null){//non-numerical literal
+						newId = new TypedId(partitionId, localRowCount);
+						//updateHistogram(context, elem);
 					}
-					
-					newId = new TypedId(partitionId, localRowCount);
+					return newId;
 				}
-				
-				//write the association between IDs and Strings in a SequenceFile
+			}
+			
+			final private TypedId handleNonLiteral(Context context, Value elem){
+				increaseNonLiteralCounters(elem, context);
+				return new TypedId(partitionId, localRowCount);
+			}
+			
+			final private void increaseNonLiteralCounters(Value elem, Context context){
+				if (elem instanceof BNode){
+					context.getCounter(ELEMENT_TYPE_GROUP, "Blanks").increment(1);
+				}
+				else{
+					context.getCounter(ELEMENT_TYPE_GROUP, "URIs").increment(1);
+				}
+			}	
+			
+			final private void saveTypeIdToFile(TypedId newId, HBaseValue value, Context context) throws IOException{
 				if (newId.getType() != TypedId.NUMERICAL){
-					id2StringWriter.append(new ImmutableBytesWritable(newId.getContent()), key);
-					if (localRowCount % 1000 == 0){
+					id2StringWriter.append(new ImmutableBytesWritable(newId.getContent()), value);
+					/*if (localRowCount % 1000 == 0){
 						id2StringWriter.syncFs();
-					}
+					}*/
 					localRowCount++;
 					context.getCounter(NUMERICAL_GROUP, "NonNumericals").increment(1);
 				}
 				else{
 					context.getCounter(NUMERICAL_GROUP, "Numericals").increment(1);
 				}
-				
-				//emit (elementID, <tripleID, position>) elements
-				for (DataPair dataPair : values) {
-					context.write(newId, dataPair);
-				}										
-					
 			}
 	}
 
+	public static void main(String[] args) {		
+		Job j1;
+		try {
+			//Path p = new Path(new Path("/test1"), new Path(ID2STRING_DIR));
+			//System.out.println(p);
+		
+			if (args.length != 3){
+				System.out.println("Usage: TripleToResource <inputPath> <inputSizeEstimate in MB> <outputPath>");
+				return;
+			}
+			
+			Path input = new Path(args[0]);
+			String outputPath = args[2];
+			
+			Path resourceIds = new Path(outputPath+TripleToResource.RESOURCE_IDS_DIR);
+			
+			j1 = BulkLoad.createTripleToResourceJob(input, resourceIds, Integer.parseInt(args[1]));
+			
+			j1.getConfiguration().set("outputPath", outputPath);
+			j1.getConfiguration().set("schemaSuffix", "_TEST");
+			j1.waitForCompletion(true);
+			
+			Configuration conf = new Configuration();
+			FileSystem fs = FileSystem.get(conf);
+			Path source = new Path(resourceIds, "idStringAssoc");
+			Path id2String = new Path(outputPath+"/idStringAssoc");
+			FileUtil.copy(fs, source, fs, id2String, true, false, conf);
+			
+		} catch (NumberFormatException e) {
+			e.printStackTrace();
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+		catch (InterruptedException e) {
+			e.printStackTrace();
+		} catch (ClassNotFoundException e) {
+			e.printStackTrace();
+		}
+
+	}
 }
https://github.com/pgroth/hbase-rdf/raw/cbf0cac3f03c8dffbc415eab1421234dfc3ead24/src%2Fmain%2Fjava%2Fnl%2Fvu%2Fdatalayer%2Fhbase%2Fbulkload%2FTripleToResource.java
58f0ae191e6421a275fb03dbf365fcaa0625e32f
modified
https://github.com/pgroth/hbase-rdf/blob/cbf0cac3f03c8dffbc415eab1421234dfc3ead24/src%2Fmain%2Fjava%2Fnl%2Fvu%2Fdatalayer%2Fhbase%2Fid%2FBaseId.java
https://api.github.com/repos/pgroth/hbase-rdf/contents/src%2Fmain%2Fjava%2Fnl%2Fvu%2Fdatalayer%2Fhbase%2Fid%2FBaseId.java?ref=cbf0cac3f03c8dffbc415eab1421234dfc3ead24
src/main/java/nl/vu/datalayer/hbase/id/BaseId.java
	public int hashCode() {//used for partitioning
	public int hashCode() {
@@ -35,7 +35,7 @@ public BaseId(){
 	}
 
 	@Override
-	public int hashCode() {
+	public int hashCode() {//used for partitioning
 		return Bytes.toInt(id, id.length-4, 4);
 	}
 
https://github.com/pgroth/hbase-rdf/raw/cbf0cac3f03c8dffbc415eab1421234dfc3ead24/src%2Fmain%2Fjava%2Fnl%2Fvu%2Fdatalayer%2Fhbase%2Fid%2FBaseId.java
d9dab74267a36bbfc3f8a7da520954ce921c8847
modified
https://github.com/pgroth/hbase-rdf/blob/cbf0cac3f03c8dffbc415eab1421234dfc3ead24/src%2Fmain%2Fjava%2Fnl%2Fvu%2Fdatalayer%2Fhbase%2Fid%2FTypedId.java
https://api.github.com/repos/pgroth/hbase-rdf/contents/src%2Fmain%2Fjava%2Fnl%2Fvu%2Fdatalayer%2Fhbase%2Fid%2FTypedId.java?ref=cbf0cac3f03c8dffbc415eab1421234dfc3ead24
src/main/java/nl/vu/datalayer/hbase/id/TypedId.java
import nl.vu.datalayer.hbase.exceptions.NumericalRangeException;

@@ -3,6 +3,8 @@
 import java.math.BigDecimal;
 import java.math.BigInteger;
 
+import nl.vu.datalayer.hbase.exceptions.NumericalRangeException;
+
 import org.apache.hadoop.hbase.util.Bytes;
 import org.openrdf.model.Literal;
 import org.openrdf.model.URI;
https://github.com/pgroth/hbase-rdf/raw/cbf0cac3f03c8dffbc415eab1421234dfc3ead24/src%2Fmain%2Fjava%2Fnl%2Fvu%2Fdatalayer%2Fhbase%2Fid%2FTypedId.java
2ceb158462467e681b1977858dd28aa29a98b0ad
modified
https://github.com/pgroth/hbase-rdf/blob/cbf0cac3f03c8dffbc415eab1421234dfc3ead24/src%2Fmain%2Fjava%2Fnl%2Fvu%2Fdatalayer%2Fhbase%2Fschema%2FHBPrefixMatchSchema.java
https://api.github.com/repos/pgroth/hbase-rdf/contents/src%2Fmain%2Fjava%2Fnl%2Fvu%2Fdatalayer%2Fhbase%2Fschema%2FHBPrefixMatchSchema.java?ref=cbf0cac3f03c8dffbc415eab1421234dfc3ead24
src/main/java/nl/vu/datalayer/hbase/schema/HBPrefixMatchSchema.java
	//private SortedMap<Short, Long> prefixCounters = null;
		//prefixCounters = prefixCounter;
	@Override
	public void create() throws Exception {
		HBaseAdmin admin = ((NativeJavaConnection)con).getAdmin();
	
		//distribute the regions over the entire ID space for String2ID
		byte[][] splits = getString2IdSplits();
		
		createTable(admin, STRING2ID+schemaSuffix, splits, false);
		
		splits = getNonNumericalSplits(NUM_REGIONS, 0);
		
		System.out.println("Id2String splits: ");
		printSplits(splits);
		
		createTable(admin, ID2STRING+schemaSuffix, splits, true);
		
		createTable(admin, TABLE_NAMES[SPOC]+schemaSuffix, splits, false);
		
		createTable(admin, TABLE_NAMES[POCS]+schemaSuffix, splits, false);
		
		byte [][] objectSplits = getObjectPrefixSplits(NUM_REGIONS);
		printSplits(objectSplits);
		createTable(admin, TABLE_NAMES[OSPC]+schemaSuffix, objectSplits, false);
		
		if (onlyTriples == false){
			createTable(admin, TABLE_NAMES[CSPO]+schemaSuffix, splits, false);
			createTable(admin, TABLE_NAMES[CPSO]+schemaSuffix, splits, false);	
			
			createTable(admin, TABLE_NAMES[OCSP]+schemaSuffix, objectSplits, false);
		}
	}
	
	/**
	 * Creates an HBase table with specified splits and optional compression for the values in the table
	 * 
	 * @param admin
	 * @param tableName
	 * @param splits
	 * @param enableCompression
	 * @throws IOException
	 */
	public static void createTable(HBaseAdmin admin, String tableName, byte [][]splits, boolean enableCompression) throws IOException{
		if (admin.tableExists(tableName) == false){
			HTableDescriptor desc = new HTableDescriptor(tableName);
			HColumnDescriptor famDesc = new HColumnDescriptor(COLUMN_FAMILY);
			
			famDesc.setBloomFilterType(StoreFile.BloomType.ROW);
			famDesc.setMaxVersions(1);
			if (enableCompression)//by default it is disabled
				famDesc.setCompactionCompressionType(Algorithm.LZO);
			desc.addFamily(famDesc);
			desc.setMaxFileSize(10*1024*1024*1024);//set maximum StoreFile size to a high value-10GB
												//so that the splits are not triggered automatically by HBase

			System.out.println("Creating table: " + tableName);
			admin.createTable(desc, splits);
		}
	}
	
	 * Divide the space between startKey and endKey into an equal amount of regions
	 * 
	
	/**
	 * Splits the space between startPartition and lastPartition into equal parts
	 * 
	 * @param numRegions
	 * @param startOffset - 0 or 1
	 * @return
	 */
	final public byte [][]getNonNumericalSplits(int numRegions, int startOffset){
		byte []startKey = new byte[KEY_LENGTH];
		byte []endKey = new byte[KEY_LENGTH];
		Arrays.fill(endKey, startOffset+Long.SIZE/8, KEY_LENGTH, (byte)0xff);
		
		long startKeyPrefix = startPartition << 24;
		Bytes.putLong(startKey, startOffset, startKeyPrefix);
		long endKeyPrefix = ((startPartition+numInputPartitions-1) << 24) | 0xffffffL;;
		Bytes.putLong(endKey, startOffset, endKeyPrefix);
		
		System.out.println("Start: "+String.format("%x", startKeyPrefix)+"; End: "+String.format("%x", endKeyPrefix));
		
		return getSplits(startKey, endKey, numRegions);
	}
	
	/**
	 * Splits the space between 0x80 and 0xff in equal parts
	 * 
	 * @param numRegions
	 * @return
	 */
	final public byte [][]getNumericalSplits(int numRegions){
		byte []startKey = new byte[KEY_LENGTH];
		startKey[0] = (byte)0x80;		
		byte []endKey = new byte[KEY_LENGTH];
		Arrays.fill(endKey, (byte)0xff);
		
		return getSplits(startKey, endKey, numRegions);
	}
	
	/**
	 * @param numRegions
	 * @return
	 */
	final private byte [][]getObjectPrefixSplits(int numRegions){
		int numericalRegions = (int)Math.round((double)numRegions*((double)numericalCount/(double)(totalStringCount+numericalCount)));
		
		byte [][]nonNumSplits = getNonNumericalSplits(numRegions-numericalRegions, 1);
		byte [][]numSplits = getNumericalSplits(numericalRegions);
		
		byte [][]splits = new byte[numRegions-1][];
		for (int i = 0; i < nonNumSplits.length; i++) {
			splits[i] = nonNumSplits[i];
		}
		
		if (nonNumSplits.length < numRegions-1){
			splits[nonNumSplits.length] = new byte[KEY_LENGTH];
			splits[nonNumSplits.length][0] = (byte)0x80;//place the key between the numerical and non-numerical splits
			for (int i = 0; i < numSplits.length; i++) {
				splits[i+nonNumSplits.length+1] = numSplits[i];
			}
		}
		return splits;
	}
	
	final private void printSplits(byte [][]splits){
		System.out.println("Number of splits: "+splits.length);
		for (int i = 0; i < splits.length; i++) {
			for (int j = 0; j < splits[i].length; j++) {
				System.out.print(String.format("%02x ", splits[i][j]));
			}
			System.out.println();
		}
	}

	final private byte[][] getString2IdSplits() {
		byte []startKey = new byte[KEY_LENGTH];
		byte []endKey = new byte[KEY_LENGTH];
		Arrays.fill(endKey, 0, KEY_LENGTH, (byte)0xff);
		byte [][]splits = getSplits(startKey, endKey, NUM_REGIONS);
		System.out.println(" String2Id splits: ");
		printSplits(splits);
		return splits;
	}

	
	/*public static BigInteger getAddress(BigInteger range, BigInteger startAddress, long chunkCount, long totalCount ){
	/*public static byte [] convertBigIntegerToAddress(BigInteger address, int length){
	}*/
	/*public byte [][] getString2IdSplits(int numRegions){
	}*/
int i = 0; i < numSplits.length; i++
src/main/java/nl/vu/datalayer/hbase/schema/HBPrefixMatchSchema.java
splits[i+nonNumSplits.length+1] = numSplits[i];
splits
			for (int i = 0; i < numSplits.length; i++)
byte [][]splits
src/main/java/nl/vu/datalayer/hbase/schema/HBPrefixMatchSchema.java
System.out.println("Number of splits: "+splits.length);
		for (int i = 0; i < splits.length; i++) {
			for (int j = 0; j < splits[i].length; j++) {
				System.out.print(String.format("%02x ", splits[i][j]));
void printSplits(byte [][]splits)
BigInteger range  BigInteger startAddress  long chunkCount  long totalCount 
src/main/java/nl/vu/datalayer/hbase/schema/HBPrefixMatchSchema.java
/*public static byte [] convertBigIntegerToAddress(BigInteger address, int length){
BigInteger getAddress(BigInteger range, BigInteger startAddress, long chunkCount, long totalCount )
	//public static final String []TABLE_NAMES = {"SPOCLUBM", "POCSLUBM", "OCSPLUBM", "CSPOLUBM", "CPSOLUBM", "OSPCLUBM"};
	private SortedMap<Short, Long> prefixCounters = null;
									SortedMap<Short, Long> prefixCounter,
		prefixCounters = prefixCounter;
	/**
	 * Creates an HBase table with specified splits and optional compression for the values in the table
	 * 
	 * @param admin
	 * @param tableName
	 * @param splits
	 * @param enableCompression
	 * @throws IOException
	 */
	public static void createTable(HBaseAdmin admin, String tableName, byte [][]splits, boolean enableCompression) throws IOException{
		if (admin.tableExists(tableName) == false){
			HTableDescriptor desc = new HTableDescriptor(tableName);

			HColumnDescriptor famDesc = new HColumnDescriptor(COLUMN_FAMILY);
			famDesc.setBloomFilterType(StoreFile.BloomType.ROW);
			famDesc.setMaxVersions(1);
			if (enableCompression)//by default it is disabled
				famDesc.setCompactionCompressionType(Algorithm.LZO);
			desc.addFamily(famDesc);

			System.out.println("Creating table: " + tableName);
			admin.createTable(desc, splits);
		}
	}
	
	public static BigInteger getAddress(BigInteger range, BigInteger startAddress, long chunkCount, long totalCount ){
	public static byte [] convertBigIntegerToAddress(BigInteger address, int length){
	}
	public byte [][] getString2IdSplits(int numRegions){
	}
	
	/**
	 * Splits the space between 0x80 and 0xff in equal parts
	 * 
	 * @param numRegions
	 * @param startOffset - 0 or 1
	 * @return
	 */
	public byte [][]getNonNumericalSplits(int numRegions, int startOffset){
		byte []startKey = new byte[KEY_LENGTH];
		byte []endKey = new byte[KEY_LENGTH];
		Arrays.fill(endKey, startOffset+Long.SIZE/8, KEY_LENGTH, (byte)0xff);
		
		long startKeyPrefix = startPartition << 24;
		Bytes.putLong(startKey, startOffset, startKeyPrefix);
		long endKeyPrefix = ((startPartition+numInputPartitions-1) << 24) | 0xffffffL;;
		Bytes.putLong(endKey, startOffset, endKeyPrefix);
		
		System.out.println("Start: "+String.format("%x", startKeyPrefix)+"; End: "+String.format("%x", endKeyPrefix));
		
		return getSplits(startKey, endKey, numRegions);
	}
	
	/**
	 * @param numRegions
	 * @return
	 */
	public byte [][]getNumericalSplits(int numRegions){
		byte []startKey = new byte[KEY_LENGTH];
		startKey[0] = (byte)0x80;		
		byte []endKey = new byte[KEY_LENGTH];
		Arrays.fill(endKey, (byte)0xff);
		
		return getSplits(startKey, endKey, numRegions);
	}
	
	/**
	 * @param numRegions
	 * @return
	 */
	public byte [][]getObjectPrefixSplits(int numRegions){
		int numericalRegions = (int)Math.round((double)numRegions*((double)numericalCount/(double)(totalStringCount+numericalCount)));
		
		byte [][]nonNumSplits = getNonNumericalSplits(numRegions-numericalRegions, 1);
		byte [][]numSplits = getNumericalSplits(numericalRegions);
		
		byte [][]splits = new byte[numRegions-1][];
		for (int i = 0; i < nonNumSplits.length; i++) {
			splits[i] = nonNumSplits[i];
		}
		
		if (nonNumSplits.length < numRegions-1){
			splits[nonNumSplits.length] = new byte[KEY_LENGTH];
			splits[nonNumSplits.length][0] = (byte)0x80;
			for (int i = 0; i < numSplits.length; i++) {
				splits[i+nonNumSplits.length+1] = numSplits[i];
			}
		}
		return splits;
	}
	
	private void printSplits(byte [][]splits){
		System.out.println("Number of splits: "+splits.length);
		for (int i = 0; i < splits.length; i++) {
			for (int j = 0; j < splits[i].length; j++) {
				System.out.print(String.format("%02x ", splits[i][j]));
			}
			System.out.println();
		}
	}

	@Override
	public void create() throws Exception {
		HBaseAdmin admin = ((NativeJavaConnection)con).getAdmin();
		//distribute the regions over the entire ID space for String2ID
		byte [][]splits = getString2IdSplits(NUM_REGIONS);
		System.out.println(" String2Id splits: ");
		printSplits(splits);
		
		createTable(admin, STRING2ID+schemaSuffix, splits, false);
		
		splits = getNonNumericalSplits(NUM_REGIONS, 0);
		
		System.out.println("Id2String splits: ");
		printSplits(splits);
		
		createTable(admin, ID2STRING+schemaSuffix, splits, true);
		
		createTable(admin, TABLE_NAMES[SPOC]+schemaSuffix, splits, false);
		
		createTable(admin, TABLE_NAMES[POCS]+schemaSuffix, splits, false);
		
		byte [][] objectSplits = getObjectPrefixSplits(NUM_REGIONS);
		printSplits(objectSplits);
		createTable(admin, TABLE_NAMES[OSPC]+schemaSuffix, objectSplits, false);
		
		if (onlyTriples == false){
			createTable(admin, TABLE_NAMES[CSPO]+schemaSuffix, splits, false);
			createTable(admin, TABLE_NAMES[CPSO]+schemaSuffix, splits, false);	
			
			createTable(admin, TABLE_NAMES[OCSP]+schemaSuffix, objectSplits, false);
		}
	}

BigInteger range  BigInteger startAddress  long chunkCount  long totalCount 
src/main/java/nl/vu/datalayer/hbase/schema/HBPrefixMatchSchema.java
public static byte [] convertBigIntegerToAddress(BigInteger address, int length){
BigInteger getAddress(BigInteger range, BigInteger startAddress, long chunkCount, long totalCount )
byte [][]splits
src/main/java/nl/vu/datalayer/hbase/schema/HBPrefixMatchSchema.java
System.out.println("Number of splits: "+splits.length);
		for (int i = 0; i < splits.length; i++) {
			for (int j = 0; j < splits[i].length; j++) {
				System.out.print(String.format("%02x ", splits[i][j]));
void printSplits(byte [][]splits)
@@ -33,7 +33,6 @@ public class HBPrefixMatchSchema implements IHBaseSchema {
 	public static final byte [] COLUMN_NAME = "".getBytes();
 	
 	public static final String []TABLE_NAMES = {"SPOC", "POCS", "OCSP", "CSPO", "CPSO", "OSPC"};
-	//public static final String []TABLE_NAMES = {"SPOCLUBM", "POCSLUBM", "OCSPLUBM", "CSPOLUBM", "CPSOLUBM", "OSPCLUBM"};
 	
 	//3 BaseIds + 1 TypedId (in the Object position of each table)
 	public static final int KEY_LENGTH = 33;
@@ -77,7 +76,7 @@ public class HBPrefixMatchSchema implements IHBaseSchema {
 	 * Histogram that stores the frequency of the last byte in each quad element 
 	 * used for proper splits for the table String2Id
 	 */
-	private SortedMap<Short, Long> prefixCounters = null;
+	//private SortedMap<Short, Long> prefixCounters = null;
 	
 	/**
 	 * Number of non-numerical elements: includes URIs, bNodes and non-numerical Literals
@@ -107,12 +106,11 @@ public HBPrefixMatchSchema(HBaseConnection con, String schemaSuffix) {
 	 */
 	public void setTableSplitInfo(long totalNumberOfStrings, long numericalParam,
 									int numPartitions, long startPartitionParam, 
-									SortedMap<Short, Long> prefixCounter,
 									boolean onlyTriplesParam){
 		startPartition = startPartitionParam;
 		numInputPartitions = numPartitions;
 		totalStringCount = totalNumberOfStrings;
-		prefixCounters = prefixCounter;
+		//prefixCounters = prefixCounter;
 		numericalCount = numericalParam;
 		onlyTriples = onlyTriplesParam; 
 	}
@@ -132,31 +130,6 @@ public static void setObjectPrefixTableSplitInfo(long totalNumberOfString, long
 		totalStringCount = totalNumberOfString;
 	}*/
 	
-	/**
-	 * Creates an HBase table with specified splits and optional compression for the values in the table
-	 * 
-	 * @param admin
-	 * @param tableName
-	 * @param splits
-	 * @param enableCompression
-	 * @throws IOException
-	 */
-	public static void createTable(HBaseAdmin admin, String tableName, byte [][]splits, boolean enableCompression) throws IOException{
-		if (admin.tableExists(tableName) == false){
-			HTableDescriptor desc = new HTableDescriptor(tableName);
-
-			HColumnDescriptor famDesc = new HColumnDescriptor(COLUMN_FAMILY);
-			famDesc.setBloomFilterType(StoreFile.BloomType.ROW);
-			famDesc.setMaxVersions(1);
-			if (enableCompression)//by default it is disabled
-				famDesc.setCompactionCompressionType(Algorithm.LZO);
-			desc.addFamily(famDesc);
-
-			System.out.println("Creating table: " + tableName);
-			admin.createTable(desc, splits);
-		}
-	}
-	
 	public void createCounterTable(HBaseAdmin admin) throws IOException{
 		
 		String fullName = COUNTER_TABLE+schemaSuffix;
@@ -247,7 +220,68 @@ public static Long getLastCounter(Configuration conf, String schemaSuffix) throw
 		return lastCounter;
 	}
 	
+	@Override
+	public void create() throws Exception {
+		HBaseAdmin admin = ((NativeJavaConnection)con).getAdmin();
+	
+		//distribute the regions over the entire ID space for String2ID
+		byte[][] splits = getString2IdSplits();
+		
+		createTable(admin, STRING2ID+schemaSuffix, splits, false);
+		
+		splits = getNonNumericalSplits(NUM_REGIONS, 0);
+		
+		System.out.println("Id2String splits: ");
+		printSplits(splits);
+		
+		createTable(admin, ID2STRING+schemaSuffix, splits, true);
+		
+		createTable(admin, TABLE_NAMES[SPOC]+schemaSuffix, splits, false);
+		
+		createTable(admin, TABLE_NAMES[POCS]+schemaSuffix, splits, false);
+		
+		byte [][] objectSplits = getObjectPrefixSplits(NUM_REGIONS);
+		printSplits(objectSplits);
+		createTable(admin, TABLE_NAMES[OSPC]+schemaSuffix, objectSplits, false);
+		
+		if (onlyTriples == false){
+			createTable(admin, TABLE_NAMES[CSPO]+schemaSuffix, splits, false);
+			createTable(admin, TABLE_NAMES[CPSO]+schemaSuffix, splits, false);	
+			
+			createTable(admin, TABLE_NAMES[OCSP]+schemaSuffix, objectSplits, false);
+		}
+	}
+	
+	/**
+	 * Creates an HBase table with specified splits and optional compression for the values in the table
+	 * 
+	 * @param admin
+	 * @param tableName
+	 * @param splits
+	 * @param enableCompression
+	 * @throws IOException
+	 */
+	public static void createTable(HBaseAdmin admin, String tableName, byte [][]splits, boolean enableCompression) throws IOException{
+		if (admin.tableExists(tableName) == false){
+			HTableDescriptor desc = new HTableDescriptor(tableName);
+			HColumnDescriptor famDesc = new HColumnDescriptor(COLUMN_FAMILY);
+			
+			famDesc.setBloomFilterType(StoreFile.BloomType.ROW);
+			famDesc.setMaxVersions(1);
+			if (enableCompression)//by default it is disabled
+				famDesc.setCompactionCompressionType(Algorithm.LZO);
+			desc.addFamily(famDesc);
+			desc.setMaxFileSize(10*1024*1024*1024);//set maximum StoreFile size to a high value-10GB
+												//so that the splits are not triggered automatically by HBase
+
+			System.out.println("Creating table: " + tableName);
+			admin.createTable(desc, splits);
+		}
+	}
+	
 	/**
+	 * Divide the space between startKey and endKey into an equal amount of regions
+	 * 
 	 * @param startKey
 	 * @param endKey
 	 * @param numRegions
@@ -276,19 +310,103 @@ public static byte[][] getSplits(byte []startKey, byte []endKey, int numRegions)
 		return splits;
 	}
 	
+	
+	/**
+	 * Splits the space between startPartition and lastPartition into equal parts
+	 * 
+	 * @param numRegions
+	 * @param startOffset - 0 or 1
+	 * @return
+	 */
+	final public byte [][]getNonNumericalSplits(int numRegions, int startOffset){
+		byte []startKey = new byte[KEY_LENGTH];
+		byte []endKey = new byte[KEY_LENGTH];
+		Arrays.fill(endKey, startOffset+Long.SIZE/8, KEY_LENGTH, (byte)0xff);
+		
+		long startKeyPrefix = startPartition << 24;
+		Bytes.putLong(startKey, startOffset, startKeyPrefix);
+		long endKeyPrefix = ((startPartition+numInputPartitions-1) << 24) | 0xffffffL;;
+		Bytes.putLong(endKey, startOffset, endKeyPrefix);
+		
+		System.out.println("Start: "+String.format("%x", startKeyPrefix)+"; End: "+String.format("%x", endKeyPrefix));
+		
+		return getSplits(startKey, endKey, numRegions);
+	}
+	
+	/**
+	 * Splits the space between 0x80 and 0xff in equal parts
+	 * 
+	 * @param numRegions
+	 * @return
+	 */
+	final public byte [][]getNumericalSplits(int numRegions){
+		byte []startKey = new byte[KEY_LENGTH];
+		startKey[0] = (byte)0x80;		
+		byte []endKey = new byte[KEY_LENGTH];
+		Arrays.fill(endKey, (byte)0xff);
+		
+		return getSplits(startKey, endKey, numRegions);
+	}
+	
+	/**
+	 * @param numRegions
+	 * @return
+	 */
+	final private byte [][]getObjectPrefixSplits(int numRegions){
+		int numericalRegions = (int)Math.round((double)numRegions*((double)numericalCount/(double)(totalStringCount+numericalCount)));
+		
+		byte [][]nonNumSplits = getNonNumericalSplits(numRegions-numericalRegions, 1);
+		byte [][]numSplits = getNumericalSplits(numericalRegions);
+		
+		byte [][]splits = new byte[numRegions-1][];
+		for (int i = 0; i < nonNumSplits.length; i++) {
+			splits[i] = nonNumSplits[i];
+		}
+		
+		if (nonNumSplits.length < numRegions-1){
+			splits[nonNumSplits.length] = new byte[KEY_LENGTH];
+			splits[nonNumSplits.length][0] = (byte)0x80;//place the key between the numerical and non-numerical splits
+			for (int i = 0; i < numSplits.length; i++) {
+				splits[i+nonNumSplits.length+1] = numSplits[i];
+			}
+		}
+		return splits;
+	}
+	
+	final private void printSplits(byte [][]splits){
+		System.out.println("Number of splits: "+splits.length);
+		for (int i = 0; i < splits.length; i++) {
+			for (int j = 0; j < splits[i].length; j++) {
+				System.out.print(String.format("%02x ", splits[i][j]));
+			}
+			System.out.println();
+		}
+	}
+
+	final private byte[][] getString2IdSplits() {
+		byte []startKey = new byte[KEY_LENGTH];
+		byte []endKey = new byte[KEY_LENGTH];
+		Arrays.fill(endKey, 0, KEY_LENGTH, (byte)0xff);
+		byte [][]splits = getSplits(startKey, endKey, NUM_REGIONS);
+		System.out.println(" String2Id splits: ");
+		printSplits(splits);
+		return splits;
+	}
+
+	
 	/**
 	 * 
 	 * @param start
 	 * @param end
 	 * @param portion
 	 */
-	public static BigInteger getAddress(BigInteger range, BigInteger startAddress, long chunkCount, long totalCount ){
+	/*public static BigInteger getAddress(BigInteger range, BigInteger startAddress, long chunkCount, long totalCount ){
 		BigInteger chunkSize = range.multiply(BigInteger.valueOf(chunkCount)).divide(BigInteger.valueOf(totalCount));
 		
 		return startAddress.add(chunkSize);
 	}
 	
-	public static byte [] convertBigIntegerToAddress(BigInteger address, int length){
+	/*public static byte [] convertBigIntegerToAddress(BigInteger address, int length){
 		byte []ret = address.toByteArray();
 		int offset = ret.length-length;
 		
@@ -300,7 +418,7 @@ else if (offset > 0){
 		}
 		else
 			return ret;
-	}
+	}*/
 	
 	
 	/**
@@ -309,7 +427,7 @@ else if (offset > 0){
 	 * @param numRegions
 	 * @return
 	 */
-	public byte [][] getString2IdSplits(int numRegions){
+	/*public byte [][] getString2IdSplits(int numRegions){
 		
 		//divide all elements to the desired number of regions, so that a proper load balance is maintained
 		if (totalStringCount < numRegions){
@@ -382,110 +500,6 @@ else if (offset > 0){
 			ret = byteSplits;
 		
 		return ret;
-	}
-	
-	/**
-	 * Splits the space between 0x80 and 0xff in equal parts
-	 * 
-	 * @param numRegions
-	 * @param startOffset - 0 or 1
-	 * @return
-	 */
-	public byte [][]getNonNumericalSplits(int numRegions, int startOffset){
-		byte []startKey = new byte[KEY_LENGTH];
-		byte []endKey = new byte[KEY_LENGTH];
-		Arrays.fill(endKey, startOffset+Long.SIZE/8, KEY_LENGTH, (byte)0xff);
-		
-		long startKeyPrefix = startPartition << 24;
-		Bytes.putLong(startKey, startOffset, startKeyPrefix);
-		long endKeyPrefix = ((startPartition+numInputPartitions-1) << 24) | 0xffffffL;;
-		Bytes.putLong(endKey, startOffset, endKeyPrefix);
-		
-		System.out.println("Start: "+String.format("%x", startKeyPrefix)+"; End: "+String.format("%x", endKeyPrefix));
-		
-		return getSplits(startKey, endKey, numRegions);
-	}
-	
-	/**
-	 * @param numRegions
-	 * @return
-	 */
-	public byte [][]getNumericalSplits(int numRegions){
-		byte []startKey = new byte[KEY_LENGTH];
-		startKey[0] = (byte)0x80;		
-		byte []endKey = new byte[KEY_LENGTH];
-		Arrays.fill(endKey, (byte)0xff);
-		
-		return getSplits(startKey, endKey, numRegions);
-	}
-	
-	/**
-	 * @param numRegions
-	 * @return
-	 */
-	public byte [][]getObjectPrefixSplits(int numRegions){
-		int numericalRegions = (int)Math.round((double)numRegions*((double)numericalCount/(double)(totalStringCount+numericalCount)));
-		
-		byte [][]nonNumSplits = getNonNumericalSplits(numRegions-numericalRegions, 1);
-		byte [][]numSplits = getNumericalSplits(numericalRegions);
-		
-		byte [][]splits = new byte[numRegions-1][];
-		for (int i = 0; i < nonNumSplits.length; i++) {
-			splits[i] = nonNumSplits[i];
-		}
-		
-		if (nonNumSplits.length < numRegions-1){
-			splits[nonNumSplits.length] = new byte[KEY_LENGTH];
-			splits[nonNumSplits.length][0] = (byte)0x80;
-			for (int i = 0; i < numSplits.length; i++) {
-				splits[i+nonNumSplits.length+1] = numSplits[i];
-			}
-		}
-		return splits;
-	}
-	
-	private void printSplits(byte [][]splits){
-		System.out.println("Number of splits: "+splits.length);
-		for (int i = 0; i < splits.length; i++) {
-			for (int j = 0; j < splits[i].length; j++) {
-				System.out.print(String.format("%02x ", splits[i][j]));
-			}
-			System.out.println();
-		}
-	}
-
-	@Override
-	public void create() throws Exception {
-		HBaseAdmin admin = ((NativeJavaConnection)con).getAdmin();
+	}*/
 	
-		//distribute the regions over the entire ID space for String2ID
-		byte [][]splits = getString2IdSplits(NUM_REGIONS);
-		System.out.println(" String2Id splits: ");
-		printSplits(splits);
-		
-		createTable(admin, STRING2ID+schemaSuffix, splits, false);
-		
-		splits = getNonNumericalSplits(NUM_REGIONS, 0);
-		
-		System.out.println("Id2String splits: ");
-		printSplits(splits);
-		
-		createTable(admin, ID2STRING+schemaSuffix, splits, true);
-		
-		createTable(admin, TABLE_NAMES[SPOC]+schemaSuffix, splits, false);
-		
-		createTable(admin, TABLE_NAMES[POCS]+schemaSuffix, splits, false);
-		
-		byte [][] objectSplits = getObjectPrefixSplits(NUM_REGIONS);
-		printSplits(objectSplits);
-		createTable(admin, TABLE_NAMES[OSPC]+schemaSuffix, objectSplits, false);
-		
-		if (onlyTriples == false){
-			createTable(admin, TABLE_NAMES[CSPO]+schemaSuffix, splits, false);
-			createTable(admin, TABLE_NAMES[CPSO]+schemaSuffix, splits, false);	
-			
-			createTable(admin, TABLE_NAMES[OCSP]+schemaSuffix, objectSplits, false);
-		}
-	}
-
 }
https://github.com/pgroth/hbase-rdf/raw/cbf0cac3f03c8dffbc415eab1421234dfc3ead24/src%2Fmain%2Fjava%2Fnl%2Fvu%2Fdatalayer%2Fhbase%2Fschema%2FHBPrefixMatchSchema.java
99327ca9dd608c10a8d80c3af908ca6242edab47
modified
https://github.com/pgroth/hbase-rdf/blob/cbf0cac3f03c8dffbc415eab1421234dfc3ead24/src%2Fmain%2Fjava%2Fnl%2Fvu%2Fdatalayer%2Fhbase%2Futil%2FHBPrefixMatchUtil.java
https://api.github.com/repos/pgroth/hbase-rdf/contents/src%2Fmain%2Fjava%2Fnl%2Fvu%2Fdatalayer%2Fhbase%2Futil%2FHBPrefixMatchUtil.java?ref=cbf0cac3f03c8dffbc415eab1421234dfc3ead24
src/main/java/nl/vu/datalayer/hbase/util/HBPrefixMatchUtil.java
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.UnsupportedEncodingException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import nl.vu.datalayer.hbase.exceptions.ElementNotFoundException;
import nl.vu.datalayer.hbase.exceptions.NumericalRangeException;
import nl.vu.datalayer.hbase.id.HBaseValue;
	private int currentTableIndex;
	/**
	 * Internal map that stores associations between Ids in the results and the corresponding Value objects
	 */
	
	private MessageDigest mDigest;

	private int keySize;

	private byte [] numericalElement;

	private String currentPattern;

	private HashMap<ByteArray, Integer> spocOffsetMap = new HashMap<ByteArray, Integer>();
		buildPattern2TableHashMap();
		try {
			mDigest = MessageDigest.getInstance("MD5");
		} catch (NoSuchAlgorithmException e) {
			e.printStackTrace();
		}
	private void buildPattern2TableHashMap(){
			throws IOException {//we expect the pattern in the order SPOC	
			retrievalInit();
			long start = System.currentTimeMillis();
			byte[] startKey = buildRangeScanKeyFromQuad(quad);
			long keyBuildOverhead = System.currentTimeMillis()-start;
			long startSearch = System.currentTimeMillis();
			ArrayList<Get> batchIdGets = doRangeScan(startKey);
			start = System.currentTimeMillis();
			Result[] id2StringResults = doBatchId2String(batchIdGets);
			System.out.println("Search time: "+searchTime+"; Id2StringOverhead: "+id2StringOverhead+"; String2IdOverhead: "
									+string2IdOverhead+"; KeyBuildOverhead: "+keyBuildOverhead);
			updateId2ValueMap(id2StringResults);			
			return buildSPOCOrderResults();
		
			System.err.println("Bound variable numerical not in expected range: "+e.getMessage());
			return null;
		} catch (ElementNotFoundException e) {
			System.err.println(e.getMessage());

	final private ArrayList<ArrayList<Value>> buildSPOCOrderResults() {
		int queryElemNo = boundElements.size();
		ArrayList<ArrayList<Value>> ret = new ArrayList<ArrayList<Value>>();
		
		for (ArrayList<ByteArray> quadList : quadResults) {
			ArrayList<Value> newQuadResult = new ArrayList<Value>(4);
			newQuadResult.addAll(Arrays.asList(new Value[]{null, null, null, null}));			
			
			fillInUnboundElements(queryElemNo, quadList, newQuadResult);		
			fillInBoundElements(newQuadResult);
			
			ret.add(newQuadResult);
		}
		return ret;
	}

	final private void fillInBoundElements(ArrayList<Value> newQuadResult) {
		for (int i = 0, j = 0; i<newQuadResult.size() && j<boundElements.size(); i++) {
			if (newQuadResult.get(i) == null){
				newQuadResult.set(i, boundElements.get(j++));
			}
		}
	}

	final private void fillInUnboundElements(int queryElemNo, ArrayList<ByteArray> quadList, ArrayList<Value> newQuadResult) {
		for (int i = 0; i < quadList.size(); i++) {
			if ((i+queryElemNo) == HBPrefixMatchSchema.ORDER[currentTableIndex][2] && 
					TypedId.getType(quadList.get(i).getBytes()[0]) == TypedId.NUMERICAL){
				//handle numericals
				TypedId id = new TypedId(quadList.get(i).getBytes());
				newQuadResult.set(2, id.toLiteral());
			}
			else{
				newQuadResult.set(HBPrefixMatchSchema.TO_SPOC_ORDER[currentTableIndex][(i+queryElemNo)], 
						id2ValueMap.get(quadList.get(i)));
			}
		}
	}

	final private void updateId2ValueMap(Result[] id2StringResults) throws IOException, ElementNotFoundException {
		HBaseValue hbaseValue = new HBaseValue();
		for (Result result : id2StringResults) {
			byte []rowVal = result.getValue(HBPrefixMatchSchema.COLUMN_FAMILY, HBPrefixMatchSchema.COLUMN_NAME);
			byte []rowKey = result.getRow();
			if (rowVal == null || rowKey == null){
				throw new ElementNotFoundException("Id not found in Id2String table: "+(rowKey == null ? null : hexaString(rowKey)));
			}
			else{
				ByteArrayInputStream byteStream = new ByteArrayInputStream(rowVal);
				hbaseValue.readFields(new DataInputStream(byteStream));
				Value val = hbaseValue.getUnderlyingValue();
				id2ValueMap.put(new ByteArray(rowKey), val);
			}
		}
	}

	final private Result[] doBatchId2String(ArrayList<Get> batchIdGets) throws IOException {
		HTableInterface id2StringTable = con.getTable(HBPrefixMatchSchema.ID2STRING+schemaSuffix);
		Result []id2StringResults = id2StringTable.get(batchIdGets);
		id2StringTable.close();
		return id2StringResults;
	}

	final private ArrayList<Get> doRangeScan(byte[] startKey) throws IOException {
		Filter prefixFilter = new PrefixFilter(startKey);
		Filter keyOnlyFilter = new KeyOnlyFilter();
		Filter filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL, prefixFilter, keyOnlyFilter);
		
		Scan scan = new Scan(startKey, filterList);
		scan.setCaching(100);

		String tableName = HBPrefixMatchSchema.TABLE_NAMES[currentTableIndex]+schemaSuffix;
		System.out.println("Retrieving from table: "+tableName);
		
		HTableInterface table = con.getTable(tableName);
		ResultScanner results = table.getScanner(scan);

		ArrayList<Get> batchGets = parseRangeScanResults(startKey.length, results);
		table.close();
		return batchGets;
	}

	final private ArrayList<Get> parseRangeScanResults(int startKeyLength, ResultScanner results) throws IOException {
		Result r = null;
		int sizeOfInterest = HBPrefixMatchSchema.KEY_LENGTH - startKeyLength;

		ArrayList<Get> batchGets = new ArrayList<Get>();
		while ((r = results.next()) != null) {
			parseKey(r.getRow(), startKeyLength, sizeOfInterest, batchGets);
		}
		results.close();
		return batchGets;
	}

	final private void retrievalInit() {
		id2ValueMap.clear();
		quadResults.clear();
		boundElements.clear();
		id2StringOverhead = 0;
		string2IdOverhead = 0;
	}
	/*public Value convertStringToValue(String val){
	}*/
		byte []md5Hash = mDigest.digest(sBytes);
		Get g = new Get(md5Hash);
			if (crtIndex == HBPrefixMatchSchema.OFFSETS[currentTableIndex][2]){//for the Object position
	final private byte []buildRangeScanKeyFromQuad(Value []quad) throws IOException, NumericalRangeException, ElementNotFoundException
		currentPattern = "";
		keySize = 0;
		spocOffsetMap.clear();
		
		ArrayList<Get> string2IdGets = buildString2IdGets(quad);	
		currentTableIndex = pattern2Table.get(currentPattern);
		
		//Query the String2Id table
		long start = System.currentTimeMillis();
		HTableInterface table = con.getTable(HBPrefixMatchSchema.STRING2ID+schemaSuffix);
		Result []results = table.get(string2IdGets);
		table.close();
		string2IdOverhead += System.currentTimeMillis()-start;
		
		return buildRangeScanKeyFromResults(results);
	}

	final private byte[] buildRangeScanKeyFromResults(Result[] results) throws ElementNotFoundException {
		byte []key = new byte[keySize];
		for (Result result : results) {
			
			byte []value = result.getValue(HBPrefixMatchSchema.COLUMN_FAMILY, HBPrefixMatchSchema.COLUMN_NAME);
			if (value == null){
				throw new ElementNotFoundException("Quad element could not be found: "+new String(result.getRow())+"\n"+hexaString(result.getRow()));
			}
			
			int spocIndex = spocOffsetMap.get(new ByteArray(result.getRow()));
			int offset = HBPrefixMatchSchema.OFFSETS[currentTableIndex][spocIndex];
			
			if (spocIndex == 2)
				Bytes.putBytes(key, offset+1, value, 0, value.length);
			else
				Bytes.putBytes(key, offset, value, 0, value.length);
		}	
		if (numericalElement != null){
			Bytes.putBytes(key, HBPrefixMatchSchema.OFFSETS[currentTableIndex][2], numericalElement, 0, numericalElement.length);
		}
		return key;
	}

	private ArrayList<Get> buildString2IdGets(Value[] quad) throws UnsupportedEncodingException, NumericalRangeException {
		ArrayList<Get> string2IdGets = new ArrayList<Get>();
		
		numericalElement = null;
				currentPattern += "?";
				currentPattern += "|";
					sBytes = quad[i].toString().getBytes("UTF-8");
				else{//Object
								numericalElement = id.getBytes();
						sBytes = lString.getBytes("UTF-8");					
					else{//bNode or URI
						sBytes = quad[i].toString().getBytes("UTF-8");
				//byte []reverseString = StringIdAssoc.reverseBytes(sBytes, sBytes.length);
				byte []md5Hash = mDigest.digest(sBytes);
				Get g = new Get(md5Hash);
				string2IdGets.add(g);
				spocOffsetMap.put(new ByteArray(md5Hash), i);
		return string2IdGets;
src/main/java/nl/vu/datalayer/hbase/util/HBPrefixMatchUtil.java
throws IOException {//we expect the pattern in the order SPOC	
			retrievalInit();
			long start = System.currentTimeMillis();
			byte[] startKey = buildRangeScanKeyFromQuad(quad);
			long keyBuildOverhead = System.currentTimeMillis()-start;
			long startSearch = System.currentTimeMillis();
			ArrayList<Get> batchIdGets = doRangeScan(startKey);
			start = System.currentTimeMillis();
			Result[] id2StringResults = doBatchId2String(batchIdGets);
			System.out.println("Search time: "+searchTime+"; Id2StringOverhead: "+id2StringOverhead+"; String2IdOverhead: "
									+string2IdOverhead+"; KeyBuildOverhead: "+keyBuildOverhead);
			updateId2ValueMap(id2StringResults);			
			return buildSPOCOrderResults();
		
			System.err.println("Bound variable numerical not in expected range: "+e.getMessage());
			return null;
void buildPattern2TableHashMap()
ArrayList<Value> newQuadResult
src/main/java/nl/vu/datalayer/hbase/util/HBPrefixMatchUtil.java
for (int i = 0, j = 0; i<newQuadResult.size() && j<boundElements.size(); i++) {
			if (newQuadResult.get(i) == null){
				newQuadResult.set(i, boundElements.get(j++));
void fillInBoundElements(ArrayList<Value> newQuadResult)
int queryElemNo  ArrayList<ByteArray> quadList  ArrayList<Value> newQuadResult
src/main/java/nl/vu/datalayer/hbase/util/HBPrefixMatchUtil.java
for (int i = 0; i < quadList.size(); i++) {
			if ((i+queryElemNo) == HBPrefixMatchSchema.ORDER[currentTableIndex][2] && 
					TypedId.getType(quadList.get(i).getBytes()[0]) == TypedId.NUMERICAL){
				//handle numericals
				TypedId id = new TypedId(quadList.get(i).getBytes());
				newQuadResult.set(2, id.toLiteral());
void fillInUnboundElements(int queryElemNo, ArrayList<ByteArray> quadList, ArrayList<Value> newQuadResult)
src/main/java/nl/vu/datalayer/hbase/util/HBPrefixMatchUtil.java
id2ValueMap.clear();
		quadResults.clear();
		boundElements.clear();
		id2StringOverhead = 0;
		string2IdOverhead = 0;
void retrievalInit()
String val
src/main/java/nl/vu/datalayer/hbase/util/HBPrefixMatchUtil.java
Value convertStringToValue(String val)
import nl.vu.datalayer.hbase.bulkload.StringIdAssoc;
import nl.vu.datalayer.hbase.id.NumericalRangeException;
	private int tableIndex;
		buildHashMap();
	private void buildHashMap(){
			throws IOException {//we expect the pattern in the order SPOC
		
			id2ValueMap.clear();
			quadResults.clear();
			boundElements.clear();
			id2StringOverhead = 0;
			byte[] startKey = buildKey(quad);
			if (startKey == null) {
				return new ArrayList<ArrayList<Value>>();
			}

			//start search in the quad tables
			long startSearch = System.currentTimeMillis();
			Filter prefixFilter = new PrefixFilter(startKey);
			Filter keyOnlyFilter = new KeyOnlyFilter();
			Filter filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL, prefixFilter, keyOnlyFilter);
			Scan scan = new Scan(startKey, filterList);
			scan.setCaching(100);

			String tableName = HBPrefixMatchSchema.TABLE_NAMES[tableIndex]+schemaSuffix;
			HTableInterface table = con.getTable(tableName);
			ResultScanner results = table.getScanner(scan);

			Result r = null;
			int sizeOfInterest = HBPrefixMatchSchema.KEY_LENGTH - startKey.length;
			HTableInterface id2StringTable = con.getTable(HBPrefixMatchSchema.ID2STRING+schemaSuffix);
			ArrayList<Get> batchGets = new ArrayList<Get>();
			while ((r = results.next()) != null) {
				parseKey(r.getRow(), startKey.length, sizeOfInterest, batchGets);
			}
			results.close();
			table.close();
			long start = System.currentTimeMillis();
			Result []id2StringResults = id2StringTable.get(batchGets);
			id2StringTable.close();
			//System.out.println("Search time: "+searchTime+"; Id2StringOverhead: "+id2StringOverhead+"; String2IdOverhead: "+string2IdOverhead);
			//update the internal mapping between ids and strings
			for (Result result : id2StringResults) {
				byte []rowVal = result.getValue(HBPrefixMatchSchema.COLUMN_FAMILY, HBPrefixMatchSchema.COLUMN_NAME);
				byte []rowKey = result.getRow();
				if (rowVal == null || rowKey == null){
					System.err.println("Id not found: "+(rowKey == null ? null : hexaString(rowKey)));
				}
				else{
					Value val = convertStringToValue(new String(rowVal));
					id2ValueMap.put(new ByteArray(rowKey), val);
				}
			}			
			
			int queryElemNo = boundElements.size();
			// build the quads in SPOC order using strings
			ArrayList<ArrayList<Value>> ret = new ArrayList<ArrayList<Value>>();
			
			for (ArrayList<ByteArray> quadList : quadResults) {
				//ArrayList<String> newList = new ArrayList<String>(quadList.size());
				
				ArrayList<Value> newQuadResult = new ArrayList<Value>(4);
				newQuadResult.addAll(Arrays.asList(new Value[]{null, null, null, null}));			
				
				//fill in unbound elements
				for (int i = 0; i < quadList.size(); i++) {
					if ((i+queryElemNo) == HBPrefixMatchSchema.ORDER[tableIndex][2] && 
							TypedId.getType(quadList.get(i).getBytes()[0]) == TypedId.NUMERICAL){
						//handle numerical
						TypedId id = new TypedId(quadList.get(i).getBytes());
						newQuadResult.set(2, id.toLiteral());
					}
					else{
						newQuadResult.set(HBPrefixMatchSchema.TO_SPOC_ORDER[tableIndex][(i+queryElemNo)], id2ValueMap.get(quadList.get(i)));
					}
				}
					
				//fill in bound elements
				for (int i = 0, j = 0; i<newQuadResult.size() && j<boundElements.size(); i++) {
					if (newQuadResult.get(i) == null){
						newQuadResult.set(i, boundElements.get(j++));
					}
				}
				
				ret.add(newQuadResult);
			}
			
			return ret;
			e.printStackTrace();
	public Value convertStringToValue(String val){
	}
		byte []key = StringIdAssoc.reverseBytes(sBytes, sBytes.length);
		Get g = new Get(key);
			if (crtIndex == HBPrefixMatchSchema.OFFSETS[tableIndex][2]){//for the Object position
	private byte []buildKey(Value []quad) throws IOException, NumericalRangeException
		String pattern = "";
		int keySize = 0;
		ArrayList<Get> string2Ids = new ArrayList<Get>();
		ArrayList<Integer> offsets = new ArrayList<Integer>();
		byte []numerical = null;
				pattern += "?";
				pattern += "|";
					sBytes = quad[i].toString().getBytes();
				else{
								numerical = id.getBytes();
						sBytes = lString.getBytes();					
					else{
						String elem = quad[i].toString();
						sBytes = elem.getBytes();
				byte []reverseString = StringIdAssoc.reverseBytes(sBytes, sBytes.length);
				Get g = new Get(reverseString);
				string2Ids.add(g);
				offsets.add(i);
			}
		}
		
		tableIndex = pattern2Table.get(pattern);
		
		//HTable table = con.getTable(HBPrefixMatchSchema.STRING2ID);
		HTableInterface table = con.getTable(HBPrefixMatchSchema.STRING2ID+schemaSuffix);
		
		byte []key = new byte[keySize];
		for (int i=0; i<string2Ids.size(); i++) {
			Get get = string2Ids.get(i);
			
			long start = System.currentTimeMillis();
			Result result = table.get(get);
			byte []value = result.getValue(HBPrefixMatchSchema.COLUMN_FAMILY, HBPrefixMatchSchema.COLUMN_NAME);
			string2IdOverhead += System.currentTimeMillis()-start;
			
			if (value == null){
				System.err.println("Quad element could not be found: "+new String(get.getRow())+"\n"+hexaString(get.getRow()));
				return null;
			
			int spocIndex = offsets.get(i);
			int offset = HBPrefixMatchSchema.OFFSETS[tableIndex][spocIndex];
			
			if (spocIndex == 2)
				Bytes.putBytes(key, offset+1, value, 0, value.length);
			else
				Bytes.putBytes(key, offset, value, 0, value.length);
		}	
		table.close();
		if (numerical != null){
			Bytes.putBytes(key, HBPrefixMatchSchema.OFFSETS[tableIndex][2], numerical, 0, numerical.length);
		return key;
src/main/java/nl/vu/datalayer/hbase/util/HBPrefixMatchUtil.java
throws IOException {//we expect the pattern in the order SPOC
		
			id2ValueMap.clear();
			quadResults.clear();
			boundElements.clear();
			id2StringOverhead = 0;
			byte[] startKey = buildKey(quad);
			if (startKey == null) {
				return new ArrayList<ArrayList<Value>>();
void buildHashMap()
Result result : id2StringResults
src/main/java/nl/vu/datalayer/hbase/util/HBPrefixMatchUtil.java
byte []rowVal = result.getValue(HBPrefixMatchSchema.COLUMN_FAMILY, HBPrefixMatchSchema.COLUMN_NAME);
				byte []rowKey = result.getRow();
				if (rowVal == null || rowKey == null){
					System.err.println("Id not found: "+(rowKey == null ? null : hexaString(rowKey)));
strings
			for (Result result : id2StringResults)
String val
src/main/java/nl/vu/datalayer/hbase/util/HBPrefixMatchUtil.java
Value convertStringToValue(String val)
@@ -1,16 +1,22 @@
 package nl.vu.datalayer.hbase.util;
 
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Properties;
 
-import nl.vu.datalayer.hbase.bulkload.StringIdAssoc;
 import nl.vu.datalayer.hbase.connection.HBaseConnection;
+import nl.vu.datalayer.hbase.exceptions.ElementNotFoundException;
+import nl.vu.datalayer.hbase.exceptions.NumericalRangeException;
 import nl.vu.datalayer.hbase.id.BaseId;
-import nl.vu.datalayer.hbase.id.NumericalRangeException;
+import nl.vu.datalayer.hbase.id.HBaseValue;
 import nl.vu.datalayer.hbase.id.TypedId;
 import nl.vu.datalayer.hbase.schema.HBPrefixMatchSchema;
 
@@ -47,7 +53,7 @@ public class HBPrefixMatchUtil implements IHBaseUtil {
 	/**
 	 * Internal use: the index of the table used for retrieval
 	 */
-	private int tableIndex;
+	private int currentTableIndex;
 	
 	/**
 	 * Variable storing time for the overhead of id2StringMap mappings upon retrieval
@@ -59,6 +65,9 @@ public class HBPrefixMatchUtil implements IHBaseUtil {
 	 */
 	private long string2IdOverhead = 0;
 	
+	/**
+	 * Internal map that stores associations between Ids in the results and the corresponding Value objects
+	 */
 	private HashMap<ByteArray, Value> id2ValueMap;
 	
 	private ArrayList<ArrayList<ByteArray>> quadResults;
@@ -68,12 +77,22 @@ public class HBPrefixMatchUtil implements IHBaseUtil {
 	private ValueFactory valueFactory;
 	
 	private String schemaSuffix;
+	
+	private MessageDigest mDigest;
+
+	private int keySize;
+
+	private byte [] numericalElement;
+
+	private String currentPattern;
+
+	private HashMap<ByteArray, Integer> spocOffsetMap = new HashMap<ByteArray, Integer>();
 
 	public HBPrefixMatchUtil(HBaseConnection con) {
 		super();
 		this.con = con;
 		pattern2Table = new HashMap<String, Integer>(16);
-		buildHashMap();
+		buildPattern2TableHashMap();
 		id2ValueMap = new HashMap<ByteArray, Value>();
 		quadResults = new ArrayList<ArrayList<ByteArray>>();
 		boundElements = new ArrayList<Value>();
@@ -87,9 +106,14 @@ public HBPrefixMatchUtil(HBaseConnection con) {
 			//continue to use the default properties
 		}
 		schemaSuffix = prop.getProperty(HBPrefixMatchSchema.SUFFIX_PROPERTY, "");	
+		try {
+			mDigest = MessageDigest.getInstance("MD5");
+		} catch (NoSuchAlgorithmException e) {
+			e.printStackTrace();
+		}
 	}
 	
-	private void buildHashMap(){
+	private void buildPattern2TableHashMap(){
 		pattern2Table.put("????", HBPrefixMatchSchema.SPOC);
 		pattern2Table.put("|???", HBPrefixMatchSchema.SPOC);
 		pattern2Table.put("||??", HBPrefixMatchSchema.SPOC);
@@ -120,105 +144,141 @@ public ArrayList<ArrayList<String>> getRow(String[] quad){
 	
 	@Override
 	public ArrayList<ArrayList<Value>> getResults(Value[] quad)
-			throws IOException {//we expect the pattern in the order SPOC
-		
+			throws IOException {//we expect the pattern in the order SPOC	
 		try {
-			id2ValueMap.clear();
-			quadResults.clear();
-			boundElements.clear();
-			id2StringOverhead = 0;
-			byte[] startKey = buildKey(quad);
-			if (startKey == null) {
-				return new ArrayList<ArrayList<Value>>();
-			}
-
-			//start search in the quad tables
-			long startSearch = System.currentTimeMillis();
-			Filter prefixFilter = new PrefixFilter(startKey);
-			Filter keyOnlyFilter = new KeyOnlyFilter();
-			Filter filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL, prefixFilter, keyOnlyFilter);
+			retrievalInit();
 			
-			Scan scan = new Scan(startKey, filterList);
-			scan.setCaching(100);
-
-			String tableName = HBPrefixMatchSchema.TABLE_NAMES[tableIndex]+schemaSuffix;
-			HTableInterface table = con.getTable(tableName);
-			ResultScanner results = table.getScanner(scan);
-
-			Result r = null;
-			int sizeOfInterest = HBPrefixMatchSchema.KEY_LENGTH - startKey.length;
-			HTableInterface id2StringTable = con.getTable(HBPrefixMatchSchema.ID2STRING+schemaSuffix);
+			long start = System.currentTimeMillis();
+			byte[] startKey = buildRangeScanKeyFromQuad(quad);
+			long keyBuildOverhead = System.currentTimeMillis()-start;
 
-			ArrayList<Get> batchGets = new ArrayList<Get>();
-			while ((r = results.next()) != null) {
-				parseKey(r.getRow(), startKey.length, sizeOfInterest, batchGets);
-			}
-			results.close();
-			table.close();
+			long startSearch = System.currentTimeMillis();
+			ArrayList<Get> batchIdGets = doRangeScan(startKey);
 			long searchTime = System.currentTimeMillis() - startSearch;
 
-			long start = System.currentTimeMillis();
-			Result []id2StringResults = id2StringTable.get(batchGets);
-			id2StringTable.close();
+			start = System.currentTimeMillis();
+			Result[] id2StringResults = doBatchId2String(batchIdGets);
 			id2StringOverhead = System.currentTimeMillis()-start;
 			
-			//System.out.println("Search time: "+searchTime+"; Id2StringOverhead: "+id2StringOverhead+"; String2IdOverhead: "+string2IdOverhead);
+			System.out.println("Search time: "+searchTime+"; Id2StringOverhead: "+id2StringOverhead+"; String2IdOverhead: "
+									+string2IdOverhead+"; KeyBuildOverhead: "+keyBuildOverhead);
 			
-			//update the internal mapping between ids and strings
-			for (Result result : id2StringResults) {
-				byte []rowVal = result.getValue(HBPrefixMatchSchema.COLUMN_FAMILY, HBPrefixMatchSchema.COLUMN_NAME);
-				byte []rowKey = result.getRow();
-				if (rowVal == null || rowKey == null){
-					System.err.println("Id not found: "+(rowKey == null ? null : hexaString(rowKey)));
-				}
-				else{
-					Value val = convertStringToValue(new String(rowVal));
-					id2ValueMap.put(new ByteArray(rowKey), val);
-				}
-			}			
-			
-			int queryElemNo = boundElements.size();
-			// build the quads in SPOC order using strings
-			ArrayList<ArrayList<Value>> ret = new ArrayList<ArrayList<Value>>();
-			
-			for (ArrayList<ByteArray> quadList : quadResults) {
-				//ArrayList<String> newList = new ArrayList<String>(quadList.size());
-				
-				ArrayList<Value> newQuadResult = new ArrayList<Value>(4);
-				newQuadResult.addAll(Arrays.asList(new Value[]{null, null, null, null}));			
-				
-				//fill in unbound elements
-				for (int i = 0; i < quadList.size(); i++) {
-					if ((i+queryElemNo) == HBPrefixMatchSchema.ORDER[tableIndex][2] && 
-							TypedId.getType(quadList.get(i).getBytes()[0]) == TypedId.NUMERICAL){
-						//handle numerical
-						TypedId id = new TypedId(quadList.get(i).getBytes());
-						newQuadResult.set(2, id.toLiteral());
-					}
-					else{
-						newQuadResult.set(HBPrefixMatchSchema.TO_SPOC_ORDER[tableIndex][(i+queryElemNo)], id2ValueMap.get(quadList.get(i)));
-					}
-				}
-					
-				//fill in bound elements
-				for (int i = 0, j = 0; i<newQuadResult.size() && j<boundElements.size(); i++) {
-					if (newQuadResult.get(i) == null){
-						newQuadResult.set(i, boundElements.get(j++));
-					}
-				}
-				
-				ret.add(newQuadResult);
-			}
-			
-			return ret;
+			updateId2ValueMap(id2StringResults);			
 			
+			return buildSPOCOrderResults();
+		
 		} catch (NumericalRangeException e) {
-			e.printStackTrace();
+			System.err.println("Bound variable numerical not in expected range: "+e.getMessage());
+			return null;
+		} catch (ElementNotFoundException e) {
+			System.err.println(e.getMessage());
 			return null;
 		}
 	}
+
+	final private ArrayList<ArrayList<Value>> buildSPOCOrderResults() {
+		int queryElemNo = boundElements.size();
+		ArrayList<ArrayList<Value>> ret = new ArrayList<ArrayList<Value>>();
+		
+		for (ArrayList<ByteArray> quadList : quadResults) {
+			ArrayList<Value> newQuadResult = new ArrayList<Value>(4);
+			newQuadResult.addAll(Arrays.asList(new Value[]{null, null, null, null}));			
+			
+			fillInUnboundElements(queryElemNo, quadList, newQuadResult);		
+			fillInBoundElements(newQuadResult);
+			
+			ret.add(newQuadResult);
+		}
+		return ret;
+	}
+
+	final private void fillInBoundElements(ArrayList<Value> newQuadResult) {
+		for (int i = 0, j = 0; i<newQuadResult.size() && j<boundElements.size(); i++) {
+			if (newQuadResult.get(i) == null){
+				newQuadResult.set(i, boundElements.get(j++));
+			}
+		}
+	}
+
+	final private void fillInUnboundElements(int queryElemNo, ArrayList<ByteArray> quadList, ArrayList<Value> newQuadResult) {
+		for (int i = 0; i < quadList.size(); i++) {
+			if ((i+queryElemNo) == HBPrefixMatchSchema.ORDER[currentTableIndex][2] && 
+					TypedId.getType(quadList.get(i).getBytes()[0]) == TypedId.NUMERICAL){
+				//handle numericals
+				TypedId id = new TypedId(quadList.get(i).getBytes());
+				newQuadResult.set(2, id.toLiteral());
+			}
+			else{
+				newQuadResult.set(HBPrefixMatchSchema.TO_SPOC_ORDER[currentTableIndex][(i+queryElemNo)], 
+						id2ValueMap.get(quadList.get(i)));
+			}
+		}
+	}
+
+	final private void updateId2ValueMap(Result[] id2StringResults) throws IOException, ElementNotFoundException {
+		HBaseValue hbaseValue = new HBaseValue();
+		for (Result result : id2StringResults) {
+			byte []rowVal = result.getValue(HBPrefixMatchSchema.COLUMN_FAMILY, HBPrefixMatchSchema.COLUMN_NAME);
+			byte []rowKey = result.getRow();
+			if (rowVal == null || rowKey == null){
+				throw new ElementNotFoundException("Id not found in Id2String table: "+(rowKey == null ? null : hexaString(rowKey)));
+			}
+			else{
+				ByteArrayInputStream byteStream = new ByteArrayInputStream(rowVal);
+				hbaseValue.readFields(new DataInputStream(byteStream));
+				Value val = hbaseValue.getUnderlyingValue();
+				id2ValueMap.put(new ByteArray(rowKey), val);
+			}
+		}
+	}
+
+	final private Result[] doBatchId2String(ArrayList<Get> batchIdGets) throws IOException {
+		HTableInterface id2StringTable = con.getTable(HBPrefixMatchSchema.ID2STRING+schemaSuffix);
+		Result []id2StringResults = id2StringTable.get(batchIdGets);
+		id2StringTable.close();
+		return id2StringResults;
+	}
+
+	final private ArrayList<Get> doRangeScan(byte[] startKey) throws IOException {
+		Filter prefixFilter = new PrefixFilter(startKey);
+		Filter keyOnlyFilter = new KeyOnlyFilter();
+		Filter filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL, prefixFilter, keyOnlyFilter);
+		
+		Scan scan = new Scan(startKey, filterList);
+		scan.setCaching(100);
+
+		String tableName = HBPrefixMatchSchema.TABLE_NAMES[currentTableIndex]+schemaSuffix;
+		System.out.println("Retrieving from table: "+tableName);
+		
+		HTableInterface table = con.getTable(tableName);
+		ResultScanner results = table.getScanner(scan);
+
+		ArrayList<Get> batchGets = parseRangeScanResults(startKey.length, results);
+		table.close();
+		return batchGets;
+	}
+
+	final private ArrayList<Get> parseRangeScanResults(int startKeyLength, ResultScanner results) throws IOException {
+		Result r = null;
+		int sizeOfInterest = HBPrefixMatchSchema.KEY_LENGTH - startKeyLength;
+
+		ArrayList<Get> batchGets = new ArrayList<Get>();
+		while ((r = results.next()) != null) {
+			parseKey(r.getRow(), startKeyLength, sizeOfInterest, batchGets);
+		}
+		results.close();
+		return batchGets;
+	}
+
+	final private void retrievalInit() {
+		id2ValueMap.clear();
+		quadResults.clear();
+		boundElements.clear();
+		id2StringOverhead = 0;
+		string2IdOverhead = 0;
+	}
 	
-	public Value convertStringToValue(String val){
+	/*public Value convertStringToValue(String val){
 		if (val.startsWith("_:")){//bNode
 			return valueFactory.createBNode(val.substring(2));
 		}
@@ -243,7 +303,7 @@ else if (val.charAt(lastDQuote+1) == '^'){
 		}else{//URIs
 			return valueFactory.createURI(val);
 		}
-	}
+	}*/
 	
 	public static String hexaString(byte []b){
 		String ret = "";
@@ -255,9 +315,9 @@ public static String hexaString(byte []b){
 	
 	public byte []retrieveId(String s) throws IOException{
 		byte []sBytes = s.getBytes();
-		byte []key = StringIdAssoc.reverseBytes(sBytes, sBytes.length);
+		byte []md5Hash = mDigest.digest(sBytes);
 		
-		Get g = new Get(key);
+		Get g = new Get(md5Hash);
 		g.addColumn(HBPrefixMatchSchema.COLUMN_FAMILY, HBPrefixMatchSchema.COLUMN_NAME);
 		
 		HTableInterface table = con.getTable(HBPrefixMatchSchema.STRING2ID+schemaSuffix);
@@ -279,7 +339,7 @@ private void parseKey(byte []key, int startIndex, int sizeOfInterest, ArrayList<
 		for (int i = 0; i < elemNo; i++) {
 			int length;
 			byte [] elemKey;
-			if (crtIndex == HBPrefixMatchSchema.OFFSETS[tableIndex][2]){//for the Object position
+			if (crtIndex == HBPrefixMatchSchema.OFFSETS[currentTableIndex][2]){//for the Object position
 				length = TypedId.SIZE;
 				if (TypedId.getType(key[crtIndex]) == TypedId.STRING){
 					elemKey = new byte[BaseId.SIZE];
@@ -316,87 +376,94 @@ private void parseKey(byte []key, int startIndex, int sizeOfInterest, ArrayList<
 		quadResults.add(currentQuad);
 	}
 	
-	private byte []buildKey(Value []quad) throws IOException, NumericalRangeException
+	final private byte []buildRangeScanKeyFromQuad(Value []quad) throws IOException, NumericalRangeException, ElementNotFoundException
 	{
-		String pattern = "";
-		int keySize = 0;
+		currentPattern = "";
+		keySize = 0;
+		spocOffsetMap.clear();
+		
+		ArrayList<Get> string2IdGets = buildString2IdGets(quad);	
+		currentTableIndex = pattern2Table.get(currentPattern);
+		
+		//Query the String2Id table
+		long start = System.currentTimeMillis();
+		HTableInterface table = con.getTable(HBPrefixMatchSchema.STRING2ID+schemaSuffix);
+		Result []results = table.get(string2IdGets);
+		table.close();
+		string2IdOverhead += System.currentTimeMillis()-start;
+		
+		return buildRangeScanKeyFromResults(results);
+	}
+
+	final private byte[] buildRangeScanKeyFromResults(Result[] results) throws ElementNotFoundException {
+		byte []key = new byte[keySize];
+		for (Result result : results) {
+			
+			byte []value = result.getValue(HBPrefixMatchSchema.COLUMN_FAMILY, HBPrefixMatchSchema.COLUMN_NAME);
+			if (value == null){
+				throw new ElementNotFoundException("Quad element could not be found: "+new String(result.getRow())+"\n"+hexaString(result.getRow()));
+			}
+			
+			int spocIndex = spocOffsetMap.get(new ByteArray(result.getRow()));
+			int offset = HBPrefixMatchSchema.OFFSETS[currentTableIndex][spocIndex];
+			
+			if (spocIndex == 2)
+				Bytes.putBytes(key, offset+1, value, 0, value.length);
+			else
+				Bytes.putBytes(key, offset, value, 0, value.length);
+		}	
 		
-		ArrayList<Get> string2Ids = new ArrayList<Get>();
-		ArrayList<Integer> offsets = new ArrayList<Integer>();
-		byte []numerical = null;
+		if (numericalElement != null){
+			Bytes.putBytes(key, HBPrefixMatchSchema.OFFSETS[currentTableIndex][2], numericalElement, 0, numericalElement.length);
+		}
+		return key;
+	}
+
+	private ArrayList<Get> buildString2IdGets(Value[] quad) throws UnsupportedEncodingException, NumericalRangeException {
+		ArrayList<Get> string2IdGets = new ArrayList<Get>();
+		
+		numericalElement = null;
 		for (int i = 0; i < quad.length; i++) {
 			if (quad[i] == null)
-				pattern += "?";
+				currentPattern += "?";
 			else{
-				pattern += "|";
+				currentPattern += "|";
 				boundElements.add(quad[i]);
 				
 				byte []sBytes;
 				if (i != 2){//not Object
 					keySize += BaseId.SIZE;		
-					sBytes = quad[i].toString().getBytes();
+					sBytes = quad[i].toString().getBytes("UTF-8");
 				}
-				else{
+				else{//Object
 					keySize += TypedId.SIZE;
 					if (quad[i] instanceof Literal){//literal
 						Literal l = (Literal)quad[i];
 						if (l.getDatatype() != null){
 							TypedId id = TypedId.createNumerical(l);
 							if (id != null){
-								numerical = id.getBytes();
+								numericalElement = id.getBytes();
 								continue;
 							}
 						}
 						String lString = l.toString();
-						sBytes = lString.getBytes();					
+						sBytes = lString.getBytes("UTF-8");					
 					}
-					else{
-						String elem = quad[i].toString();
-						sBytes = elem.getBytes();
+					else{//bNode or URI
+						sBytes = quad[i].toString().getBytes("UTF-8");
 					}
 				}
 				
-				byte []reverseString = StringIdAssoc.reverseBytes(sBytes, sBytes.length);
-				Get g = new Get(reverseString);
+				//byte []reverseString = StringIdAssoc.reverseBytes(sBytes, sBytes.length);
+				byte []md5Hash = mDigest.digest(sBytes);
+				Get g = new Get(md5Hash);
 				g.addColumn(HBPrefixMatchSchema.COLUMN_FAMILY, HBPrefixMatchSchema.COLUMN_NAME);
-				string2Ids.add(g);
-				offsets.add(i);
-			}
-		}
-		
-		tableIndex = pattern2Table.get(pattern);
-		
-		//HTable table = con.getTable(HBPrefixMatchSchema.STRING2ID);
-		HTableInterface table = con.getTable(HBPrefixMatchSchema.STRING2ID+schemaSuffix);
-		
-		byte []key = new byte[keySize];
-		for (int i=0; i<string2Ids.size(); i++) {
-			Get get = string2Ids.get(i);
-			
-			long start = System.currentTimeMillis();
-			Result result = table.get(get);
-			byte []value = result.getValue(HBPrefixMatchSchema.COLUMN_FAMILY, HBPrefixMatchSchema.COLUMN_NAME);
-			string2IdOverhead += System.currentTimeMillis()-start;
-			
-			if (value == null){
-				System.err.println("Quad element could not be found: "+new String(get.getRow())+"\n"+hexaString(get.getRow()));
-				return null;
+				string2IdGets.add(g);
+				spocOffsetMap.put(new ByteArray(md5Hash), i);
 			}
-			
-			int spocIndex = offsets.get(i);
-			int offset = HBPrefixMatchSchema.OFFSETS[tableIndex][spocIndex];
-			
-			if (spocIndex == 2)
-				Bytes.putBytes(key, offset+1, value, 0, value.length);
-			else
-				Bytes.putBytes(key, offset, value, 0, value.length);
-		}	
-		table.close();
-		if (numerical != null){
-			Bytes.putBytes(key, HBPrefixMatchSchema.OFFSETS[tableIndex][2], numerical, 0, numerical.length);
 		}
 		
-		return key;
+		return string2IdGets;
 	}
 
 	@Override
https://github.com/pgroth/hbase-rdf/raw/cbf0cac3f03c8dffbc415eab1421234dfc3ead24/src%2Fmain%2Fjava%2Fnl%2Fvu%2Fdatalayer%2Fhbase%2Futil%2FHBPrefixMatchUtil.java
d4e8a2428225e72009e98546539b2e513b8bcc03
modified
https://github.com/pgroth/hbase-rdf/blob/cbf0cac3f03c8dffbc415eab1421234dfc3ead24/src%2Fmain%2Fjava%2Fnl%2Fvu%2Fdatalayer%2Fhbase%2Futil%2FIHBaseUtil.java
https://api.github.com/repos/pgroth/hbase-rdf/contents/src%2Fmain%2Fjava%2Fnl%2Fvu%2Fdatalayer%2Fhbase%2Futil%2FIHBaseUtil.java?ref=cbf0cac3f03c8dffbc415eab1421234dfc3ead24
src/main/java/nl/vu/datalayer/hbase/util/IHBaseUtil.java
	 * @param quad - array of Value objects with null elements in un-bound positions
	 * @return list of results
	 * 			null - in case of errors

	 * @param quad
	 * @return

@@ -18,8 +18,9 @@ public interface IHBaseUtil {
 	/**
 	 * Solves a simple query in which un-bound variable are expected to be null
 	 * 
-	 * @param quad
-	 * @return
+	 * @param quad - array of Value objects with null elements in un-bound positions
+	 * @return list of results
+	 * 			null - in case of errors
 	 * @throws IOException
 	 */
 	public ArrayList<ArrayList<Value>> getResults(Value []quad) throws IOException;

https://github.com/pgroth/hbase-rdf/raw/cbf0cac3f03c8dffbc415eab1421234dfc3ead24/src%2Fmain%2Fjava%2Fnl%2Fvu%2Fdatalayer%2Fhbase%2Futil%2FIHBaseUtil.java
b6f18c4d022e7819fd92e662662b2f75233cfc00
modified
pgroth$hbase-rdf
throws IOException {//we expect the pattern in the order SPOC	
			retrievalInit();
			long start = System.currentTimeMillis();
			byte[] startKey = buildRangeScanKeyFromQuad(quad);
			long keyBuildOverhead = System.currentTimeMillis()-start;
			long startSearch = System.currentTimeMillis();
			ArrayList<Get> batchIdGets = doRangeScan(startKey);
			start = System.currentTimeMillis();
			Result[] id2StringResults = doBatchId2String(batchIdGets);
			System.out.println("Search time: "+searchTime+"; Id2StringOverhead: "+id2StringOverhead+"; String2IdOverhead: "
									+string2IdOverhead+"; KeyBuildOverhead: "+keyBuildOverhead);
			updateId2ValueMap(id2StringResults);			
			return buildSPOCOrderResults();
		
			System.err.println("Bound variable numerical not in expected range: "+e.getMessage());
			return null;
void
void buildPattern2TableHashMap()
throws IOException {//we expect the pattern in the order SPOC
		
			id2ValueMap.clear();
			quadResults.clear();
			boundElements.clear();
			id2StringOverhead = 0;
			byte[] startKey = buildKey(quad);
			if (startKey == null) {
				return new ArrayList<ArrayList<Value>>();
void
void buildHashMap()