String[] args
src/main/java/nl/vu/datalayer/hbase/bulkload/BulkLoad.java
try {
		Path input = new Path(args[0]);
		String outputPath = args[2];
		Path resourceIds = new Path(outputPath+TripleToResource.RESOURCE_IDS_DIR);
		Path convertedTripletsPath = new Path(outputPath+ResourceToTriple.TEMP_TRIPLETS_DIR);
		
		Path string2IdInput = new Path(outputPath+TripleToResource.ID2STRING_DIR);
		Path string2IdOutput = new Path(outputPath+StringIdAssoc.STRING2ID_DIR);
		
		HBaseConnection con = new HBaseConnection(HBaseConnection.JAVA_API);
		HBPrefixMatchSchema.createCounterTable(con.getAdmin());
		
		long start = System.currentTimeMillis();
		
		Job j1 = createTripleToResourceJob(input, resourceIds, Integer.parseInt(args[1]));
		j1.getConfiguration().set("outputPath", outputPath);
		j1.waitForCompletion(true);
		
		long firstJob = System.currentTimeMillis()-start;
		System.out.println("First pass finished in "+firstJob+" ms");
		
		//retrieve counter values and build a SortedMap
		retrieveCounters(j1);
		long startPartition = HBPrefixMatchSchema.updateLastCounter(tripleToResourceReduceTasks, con.getConfiguration())+1;
void main(String[] args)
String[] args
src/main/java/nl/vu/datalayer/hbase/bulkload/BulkLoad.java
try {
			if (args.length != 5){
				System.out.println("Usage: bulkLoad <inputPath> <inputSizeEstimate in MB> <outputPath> <schemaSuffix> <onlyTriples(true/false)>");
				return;
void main(String[] args)
String[] args
src/main/java/nl/vu/datalayer/hbase/bulkload/BulkLoad.java
input = new Path(args[0]);
		inputEstimateSize = Integer.parseInt(args[1]);
		outputPath = args[2];
		schemaSuffix = args[3];
		onlyTriples = Boolean.parseBoolean(args[4]);
void initializeLocalVariables(String[] args)
Context context  Value literal
src/main/java/nl/vu/datalayer/hbase/bulkload/TripleToResource.java
byte []bytes = literal.stringValue().getBytes();
				context.getCounter(HISTOGRAM_GROUP, 
						String.format("%02x", (short)bytes[bytes.length-1] & 0xfe)).increment(1);
void updateHistogram(Context context, Value literal)
DataPair dataPair : values
src/main/java/nl/vu/datalayer/hbase/bulkload/TripleToResource.java
context.write(newId, dataPair);
				catch(NumericalRangeException e){
					System.err.println("Numerical not in expected range: "+e.getMessage());
					context.getCounter(EXCEPTION_GROUP, "RangeExceptions").increment(1);
					return;
elements
					for (DataPair dataPair : values)
Context context  Value elem
src/main/java/nl/vu/datalayer/hbase/bulkload/TripleToResource.java
increaseNonLiteralCounters(elem, context);
				return new TypedId(partitionId, localRowCount);
TypedId handleNonLiteral(Context context, Value elem)
Value elem  Context context
src/main/java/nl/vu/datalayer/hbase/bulkload/TripleToResource.java
if (elem instanceof BNode){
					context.getCounter(ELEMENT_TYPE_GROUP, "Blanks").increment(1);
void increaseNonLiteralCounters(Value elem, Context context)
String[] args
src/main/java/nl/vu/datalayer/hbase/bulkload/TripleToResource.java
Job j1;
		try {
			//Path p = new Path(new Path("/test1"), new Path(ID2STRING_DIR));
			//System.out.println(p);
		
			if (args.length != 3){
				System.out.println("Usage: TripleToResource <inputPath> <inputSizeEstimate in MB> <outputPath>");
				return;
void main(String[] args)
int i = 0; i < numSplits.length; i++
src/main/java/nl/vu/datalayer/hbase/schema/HBPrefixMatchSchema.java
splits[i+nonNumSplits.length+1] = numSplits[i];
splits
			for (int i = 0; i < numSplits.length; i++)
byte [][]splits
src/main/java/nl/vu/datalayer/hbase/schema/HBPrefixMatchSchema.java
System.out.println("Number of splits: "+splits.length);
		for (int i = 0; i < splits.length; i++) {
			for (int j = 0; j < splits[i].length; j++) {
				System.out.print(String.format("%02x ", splits[i][j]));
void printSplits(byte [][]splits)
BigInteger range  BigInteger startAddress  long chunkCount  long totalCount 
src/main/java/nl/vu/datalayer/hbase/schema/HBPrefixMatchSchema.java
/*public static byte [] convertBigIntegerToAddress(BigInteger address, int length){
BigInteger getAddress(BigInteger range, BigInteger startAddress, long chunkCount, long totalCount )
src/main/java/nl/vu/datalayer/hbase/util/HBPrefixMatchUtil.java
throws IOException {//we expect the pattern in the order SPOC	
			retrievalInit();
			long start = System.currentTimeMillis();
			byte[] startKey = buildRangeScanKeyFromQuad(quad);
			long keyBuildOverhead = System.currentTimeMillis()-start;
			long startSearch = System.currentTimeMillis();
			ArrayList<Get> batchIdGets = doRangeScan(startKey);
			start = System.currentTimeMillis();
			Result[] id2StringResults = doBatchId2String(batchIdGets);
			System.out.println("Search time: "+searchTime+"; Id2StringOverhead: "+id2StringOverhead+"; String2IdOverhead: "
									+string2IdOverhead+"; KeyBuildOverhead: "+keyBuildOverhead);
			updateId2ValueMap(id2StringResults);			
			return buildSPOCOrderResults();
		
			System.err.println("Bound variable numerical not in expected range: "+e.getMessage());
			return null;
void buildPattern2TableHashMap()
ArrayList<Value> newQuadResult
src/main/java/nl/vu/datalayer/hbase/util/HBPrefixMatchUtil.java
for (int i = 0, j = 0; i<newQuadResult.size() && j<boundElements.size(); i++) {
			if (newQuadResult.get(i) == null){
				newQuadResult.set(i, boundElements.get(j++));
void fillInBoundElements(ArrayList<Value> newQuadResult)
int queryElemNo  ArrayList<ByteArray> quadList  ArrayList<Value> newQuadResult
src/main/java/nl/vu/datalayer/hbase/util/HBPrefixMatchUtil.java
for (int i = 0; i < quadList.size(); i++) {
			if ((i+queryElemNo) == HBPrefixMatchSchema.ORDER[currentTableIndex][2] && 
					TypedId.getType(quadList.get(i).getBytes()[0]) == TypedId.NUMERICAL){
				//handle numericals
				TypedId id = new TypedId(quadList.get(i).getBytes());
				newQuadResult.set(2, id.toLiteral());
void fillInUnboundElements(int queryElemNo, ArrayList<ByteArray> quadList, ArrayList<Value> newQuadResult)
src/main/java/nl/vu/datalayer/hbase/util/HBPrefixMatchUtil.java
id2ValueMap.clear();
		quadResults.clear();
		boundElements.clear();
		id2StringOverhead = 0;
		string2IdOverhead = 0;
void retrievalInit()
String val
src/main/java/nl/vu/datalayer/hbase/util/HBPrefixMatchUtil.java
Value convertStringToValue(String val)
String[] args
src/main/java/nl/vu/datalayer/hbase/bulkload/BulkLoad.java
try {
		Path input = new Path(args[0]);
		String outputPath = args[2];
		Path resourceIds = new Path(outputPath+TripleToResource.RESOURCE_IDS_DIR);
		Path convertedTripletsPath = new Path(outputPath+ResourceToTriple.TEMP_TRIPLETS_DIR);
		
		Path string2IdInput = new Path(outputPath+TripleToResource.ID2STRING_DIR);
		Path string2IdOutput = new Path(outputPath+StringIdAssoc.STRING2ID_DIR);
		
		HBaseConnection con = new HBaseConnection(HBaseConnection.JAVA_API);
		HBPrefixMatchSchema.createCounterTable(con.getAdmin());
		
		long start = System.currentTimeMillis();
		
		Job j1 = createTripleToResourceJob(input, resourceIds, Integer.parseInt(args[1]));
		j1.getConfiguration().set("outputPath", outputPath);
		j1.waitForCompletion(true);
		long firstJob = System.currentTimeMillis()-start;
		System.out.println("First pass finished in "+firstJob+" ms");
		
		//retrieve counter values and build a SortedMap
		retrieveCounters(j1);
		long startPartition = HBPrefixMatchSchema.updateLastCounter(tripleToResourceReduceTasks, con.getConfiguration())+1;
void main(String[] args)
String[] args
src/main/java/nl/vu/datalayer/hbase/bulkload/BulkLoad.java
try {
			if (args.length != 5){
				System.out.println("Usage: bulkLoad <inputPath> <inputSizeEstimate in MB> <outputPath> <schemaSuffix> <onlyTriples(true/false)>");
				return;
			
			Path input = new Path(args[0]);
			String outputPath = args[2];
			schemaSuffix = args[3];
			boolean onlyTriples = Boolean.parseBoolean(args[4]); 
			
			Path resourceIds = new Path(outputPath+TripleToResource.RESOURCE_IDS_DIR);
			Path convertedTripletsPath = new Path(outputPath+ResourceToTriple.TEMP_TRIPLETS_DIR);
			
			Path string2IdInput = new Path(outputPath+TripleToResource.ID2STRING_DIR);
			Path string2IdOutput = new Path(outputPath+StringIdAssoc.STRING2ID_DIR);
			
			NativeJavaConnection con = (NativeJavaConnection)HBaseConnection.create(HBaseConnection.NATIVE_JAVA);
			HBPrefixMatchSchema prefMatchSchema = new HBPrefixMatchSchema(con, schemaSuffix);
			prefMatchSchema.createCounterTable(con.getAdmin());
			
			long start = System.currentTimeMillis();
			
			Job j1 = createTripleToResourceJob(input, resourceIds, Integer.parseInt(args[1]));//TODO
			j1.getConfiguration().set("outputPath", outputPath);
			j1.getConfiguration().set("schemaSuffix", schemaSuffix);
			j1.waitForCompletion(true);
			
			long firstJob = System.currentTimeMillis()-start;
			System.out.println("First pass finished in "+firstJob+" ms");
			
			//retrieve counter values and build a SortedMap
			retrieveCounters(j1);//TODO
			long startPartition = HBPrefixMatchSchema.updateLastCounter(tripleToResourceReduceTasks, con.getConfiguration(), schemaSuffix)+1;//TODO
			//long startPartition = HBPrefixMatchSchema.getLastCounter(con.getConfiguration())+1;
			
			
			//SECOND PASS-----------------------------------------------	
			
			Job j2 = createResourceToTripleJob(resourceIds, convertedTripletsPath);	
			j2.waitForCompletion(true);
			
			System.out.println("Second pass finished");
			
			//read counters from file
			//buildCountersFromFile();
			//create all tables containing PrefixMatch schema ----------------------
			System.out.println(totalStringCount+" : "+numericalCount);
					
			prefMatchSchema.setTableSplitInfo(totalStringCount, numericalCount, 
					tripleToResourceReduceTasks, startPartition, sufixCounters, 
					onlyTriples);
			prefMatchSchema.create();
			
			LoadIncrementalHFiles bulkLoad = new LoadIncrementalHFiles(con.getConfiguration());
			
			//String2Id PASS----------------------------------------------
					
			Job j3 = createString2IdJob(con, string2IdInput, string2IdOutput);
			j3.waitForCompletion(true);
			
			//string2Id = con.getTable(HBPrefixMatchSchema.STRING2ID);
			doBulkLoad(bulkLoad, string2IdOutput, string2Id);
			
			System.out.println("Finished bulk load for String2Id table");//=====================//=============
			
			Path id2StringOutput = new Path(outputPath+StringIdAssoc.ID2STRING_DIR);
			Job j4 = createId2StringJob(con, string2IdInput, id2StringOutput);
			j4.waitForCompletion(true);
			
			doBulkLoad(bulkLoad, id2StringOutput, id2String);
			
			System.out.println("Finished bulk load for Id2String table");//=====================//==================================
			
			Path spocPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.SPOC]);
			Path pocsPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.POCS]);
			Path ospcPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.OSPC]);
			
			//SPOC--------------------
			Job j5 = createPrefixMatchJob(con, convertedTripletsPath, spocPath, HBPrefixMatchSchema.SPOC, PrefixMatch.PrefixMatchSPOCMapper.class);
			j5.waitForCompletion(true);
			
			doBulkLoad(bulkLoad, spocPath, currentTable);
			if (onlyTriples == false){//the table with prefix context
				Path cspoPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.CSPO]);
				Path cpsoPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.CPSO]);
				Path ocspPath = new Path(outputPath+"/"+HBPrefixMatchSchema.TABLE_NAMES[HBPrefixMatchSchema.OCSP]);
				
				//CSPO----------------------
				Job j7 = createPrefixMatchJob(con, convertedTripletsPath, cspoPath, HBPrefixMatchSchema.CSPO, PrefixMatch.PrefixMatchCSPOMapper.class);
				j7.waitForCompletion(true);
				
				doBulkLoad(bulkLoad, cspoPath, currentTable);
				
				//CPSO-------------------------
				Job j8 = createPrefixMatchJob(con, convertedTripletsPath, cpsoPath, HBPrefixMatchSchema.CPSO, PrefixMatch.PrefixMatchCPSOMapper.class);
				j8.waitForCompletion(true);
				
				doBulkLoad(bulkLoad, cpsoPath, currentTable);
				
				//OCSP--------------------------			
				Job j9 = createPrefixMatchJob(con, convertedTripletsPath, ocspPath, HBPrefixMatchSchema.OCSP, PrefixMatch.PrefixMatchOCSPMapper.class);
				j9.waitForCompletion(true);
				
				doBulkLoad(bulkLoad, ocspPath, currentTable);
void main(String[] args)
Context context  String literal
src/main/java/nl/vu/datalayer/hbase/bulkload/TripleToResource.java
if (literal.endsWith("\"")){
					context.getCounter(HISTOGRAM_GROUP, String.format("%02x", (short)literal.getBytes()[literal.getBytes().length-2] & 0xfe)).increment(1);
void updateHistogram(Context context, String literal)
DataPair dataPair : values
src/main/java/nl/vu/datalayer/hbase/bulkload/TripleToResource.java
context.write(newId, dataPair);
elements
				for (DataPair dataPair : values)
BigInteger range  BigInteger startAddress  long chunkCount  long totalCount 
src/main/java/nl/vu/datalayer/hbase/schema/HBPrefixMatchSchema.java
public static byte [] convertBigIntegerToAddress(BigInteger address, int length){
BigInteger getAddress(BigInteger range, BigInteger startAddress, long chunkCount, long totalCount )
byte [][]splits
src/main/java/nl/vu/datalayer/hbase/schema/HBPrefixMatchSchema.java
System.out.println("Number of splits: "+splits.length);
		for (int i = 0; i < splits.length; i++) {
			for (int j = 0; j < splits[i].length; j++) {
				System.out.print(String.format("%02x ", splits[i][j]));
void printSplits(byte [][]splits)
src/main/java/nl/vu/datalayer/hbase/util/HBPrefixMatchUtil.java
throws IOException {//we expect the pattern in the order SPOC
		
			id2ValueMap.clear();
			quadResults.clear();
			boundElements.clear();
			id2StringOverhead = 0;
			byte[] startKey = buildKey(quad);
			if (startKey == null) {
				return new ArrayList<ArrayList<Value>>();
void buildHashMap()
Result result : id2StringResults
src/main/java/nl/vu/datalayer/hbase/util/HBPrefixMatchUtil.java
byte []rowVal = result.getValue(HBPrefixMatchSchema.COLUMN_FAMILY, HBPrefixMatchSchema.COLUMN_NAME);
				byte []rowKey = result.getRow();
				if (rowVal == null || rowKey == null){
					System.err.println("Id not found: "+(rowKey == null ? null : hexaString(rowKey)));
strings
			for (Result result : id2StringResults)
String val
src/main/java/nl/vu/datalayer/hbase/util/HBPrefixMatchUtil.java
Value convertStringToValue(String val)