<NME> app_br_lr.properties
<BEF> ########## IO ###########r1
# folder for all outputs
output.folder=/mnt/home/cheng/app
# the name of the train folder
output.trainFolder=train
# the name of the test folder
output.testFolder=test
# the name of the validation folder
output.validFolder=valid
# the name of the calibration folder
output.calibrationFolder=cali

output.modelFolder=lr
# users can train mulitple calibrators using different calibrator types and data sources and store them in differt folders
output.calibratorFolder=label_iso_set_reranker_wq
# the full name of the log file
# if not given, the log will be printed to the console
output.log=/mnt/home/cheng/app/log_lr

######### functions ##########
# use two queries to specify training set and test set
# the query string should be the string after the top level "query":
# For example, if the curl command is
# curl -XGET "http://localhost:9200/ohsumed_20000/document/_search" -d'{"query":{"filtered":{"query":{"match_all":{}},"filter":{"term":{"split":"train"}}}}}'
# then the query string should be {"filtered":{"query":{"match_all":{}},"filter":{"term":{"split":"train"}}}}
# make sure the curly braces match

# the elasticsearch query string for matching training documents
train.splitQuery={"bool": {"must":[{"range":{"random": {"gte": 0, "lt": 0.7}}},{"match":{"entity":"mgh"}},{"range":{"cdf":{"gte": 0, "lt": 0.95}}}]}}
# the elasticsearch query string for matching test documents
test.splitQuery={"bool":{"must":[{"range":{"random":{"gte": 0.7,"lt": 0.8}}},{"match":{"entity":"mgh"}}]}}
# the elasticsearch query string for matching validation documents
valid.splitQuery={"bool": {"must": {"match_all": {}},"filter": {"term": {"split": "valid"}}}}
# the elasticsearch query string for matching calibration documents
calibration.splitQuery={"bool": {"must": {"match_all": {}},"filter": {"term": {"split": "valid"}}}}

# generate training set
createTrainSet=true
# generate test set
createTestSet=true
# generate validation set
createValidSet=true
# generate calibration set
createCalibrationSet=true

# train model; produce reports for training set; train should be run prior to test
train=true
# calibrate probabilities
calibrate=true
# tune confidence threshold for target performance on valid set and calibration set
tuneThreshold=true
# load model; produce reports for validation set
validate=true
# load model; produce reports for test set
test=true


########## index ########## 
index.indexName=entrad-lr-common_20180101_20171001_20180101
index.clusterName=VobaModelGeneration
index.documentType=document
# node or transport
index.clientType=transport
# set hosts if clientType=transport
index.hosts=localhost
# set ports if clientType=transport
index.ports=9300

######### feature ##########
# if true, normalize all features to the same scale
train.feature.normalize=true
train.feature.useInitialFeatures=true
train.feature.featureFieldPrefix=work_queue
train.feature.categFeature.filter=false
train.feature.categFeature.percentThreshold=0.1
# if true, ngrams can have repeated words; if false, we only keep ngrams in which all words are distinct
train.feature.ngram.allowDuplicateWords=true
# if true, "a b" cannot match "b a"
train.feature.ngram.inOrder=true
train.feature.ngram.n=1,2
# how frequent an ngram needs to be in order to be considered as a feature
# the minimum fraction of documents containing the ngram; 0.02 means 2 percent
train.feature.ngram.minDf=0.001
train.feature.ngram.slop=0,1,2
train.feature.ngram.extractionFields=body,ngram_reasonforvisit
# can be es_original, frequency, binary, tfifl
# tfifl = term frequency normalized by field length;
# to use tfifl, users should manually store the field length in a separate field named <field_name>_field_length, e.g., body_field_length
train.feature.ngram.matchScoreType=tfifl

# whether to perform feature selection for ngrams
train.feature.ngram.selection=false
# if selection=true, how many ngrams to keep for each label
train.feature.ngram.selectPerLabel=100

train.feature.missingValue=false

# whether to add external ngrams from a file
# if true, user specified ngrams will be added to the feature matrix
train.feature.addExternalNgrams=false
# the file containing user specified ngrams, one ngram per line
# users do not need to manually analyze (tokenize, stem, filter) these ngrams, but need to specify the right analyzer below
train.feature.externalNgramFile=

# whether to filter ngram candidates based on a list of unigram keywords
# if true, only ngrams (n>1) containing at least one of the keywords will be kept
train.feature.filterNgramsByKeyWords=false
# the file containing unigram keywords, one unigram per line
# users do not need to manually analyze (tokenize, stem, filter) these unigrams, but need to specify the right analyzer below
train.feature.filterNgrams.keyWordsFile=

# the program will use this analyzer to analyze (tokenize, stem, filter) the user provided ngrams and unigram keywords
# this analyzer should be the same as the one specified in the Elasticsearch body field mapping
train.feature.analyzer=english

# whether to filter ngram candidates by a regular expression
# if true, ngrams matching the regular expression will be removed
train.feature.filterNgramsByRegex=true

# the regular expression used to filter ngrams
# for example, \\d will remove a unigram if it is a digit
# users can specify ngram length in the regular expression using the number of spaces (an ngram has n-1 spaces)
# to help users understand and test the regular expression matching function, pyramid provides another application called "regex"
# users are encouraged to test their regular expression first with "regex"
train.feature.filterNgrams.regex=((.*) )*\\d{3,}+( (.*))*

# whether to consider code description as features
train.feature.useCodeDescription=false
# the file containing code descriptions, each line in the file describes one code
train.feature.codeDesc.File=
# the program will use this analyzer to analyze (tokenize, stem, filter) the code descriptions
# this analyzer should be the same as the one specified in the Elasticsearch body field mapping
train.feature.codeDesc.analyzer=english
# which field we should match code descriptions against to compute matching scores
train.feature.codeDesc.matchField=body
# As most of the documents would match at least one word in the code description, direct partial matching results in dense features.
# We provide a threshold specifying the minimum percentage of the words in the code description the document must match. 
# Matching below the threshold will be discarded.
# For example, if the description has 10 terms and a document matches 2 terms and minMatchPercentage=30, the matching score will be set to 0.
# The implementation uses minimum should match query + constance score
# see https://www.elastic.co/guide/en/elasticsearch/guide/current/ignoring-tfidf.html
# Matching k out of n words will gives a score of k^2/n^1.5 
# For boosting classifiers, we can simply interpret the score as k as monotonic transformations do no matter
train.feature.codeDesc.minMatchPercentage=20

########## labels ##############
# the elasticsearch index field for labels
train.label.field=array_translated_icd10entry2
train.label.filterByPrefix=false
train.label.filter.prefix=
# encode labels in frequency or alphabetical order
train.label.order=frequency
# During training, only consider labels with document count above this (absolute) threshold; set this threshold to 0 if you want consider all labels
train.label.minDF=0
# whether to consider new test labels during evaluation; this option does not affect the actual prediction; new labels will never be predicted;
# setting this paramater to true will include new test labels into the test performance evaluation
test.considerNewLabel=true
valid.considerNewLabel=true
calibration.considerNewLabel=true

########### weights ############
# whether to incorporate instance weights in model training
train.useInstanceWeights=false
# the elasticsearch filed storing instance weights
train.weight.field=w12

#### training algorithm hyper parameters #####
train.penalty=1E-9
train.l1Ratio=0.7


############ calibration #############
# users can train mulitple calibrators using different calibrator types and data sources and store them in differt folders
# isotonic or identity
calibrate.labelCalibrator=isotonic
# isotonic, cardinality_isotonic, reranker, identity, or zero
calibrate.setCalibrator=reranker
# calibration target;
# accuracy or f1
calibrate.target=accuracy
calibrate.numCandidates=50
calibrate.reranker.numLeaves=10

calibrate.reranker.useInitialFeatures=true
calibrate.reranker.featureFieldPrefix=work_queue


############ Confidence Threshold ###############
# target value for autocoding
threshold.targetValue=0.9
# the metric used to tune threshold; for CTAT, the metric is accuracy; for ctft,the metirc is f1
# accuracy or f1
threshold.targetMetric=accuracy
# name of the file that saves confidence threshold.
# users can tune multiple thresholds and save them to different files.
threshold.name=threshold
threshold.lowerBound=0.6
threshold.upperBound=0.9 

############ prediction #########
# independent, support, reranker
predict.mode=reranker
# the minimum predicted label set size
predict.minSize=1
# the maximum predicted label set size
predict.maxSize=20

############ report ##################
report.rule.limit=10
report.numDocsPerFile=100
# When generating prediction reports for individual label probabilities, labels with probabilities below the threshold will not be displayed
# This only make the reports more readable; it does not affect the actual prediction in any way.
report.classProbThreshold=0.2
report.labelSetLimit=10
# whether to show prediction details in Json files
report.showPredictionDetail=false
# whether to produce the HTML files from Json files
# currently not supported in LR
report.produceHTML=false

# the internal Java class name for this application. 
# users do not need to modify this.
pyramid.class=AppBRLR

<MSG> fix typo in config file

<DFF> @@ -32,7 +32,7 @@ test.splitQuery={"bool": {"must": {"match_all": {}},"filter": {"term": {"split":
 # the elasticsearch query string for matching validation documents
 valid.splitQuery={"bool": {"must": {"match_all": {}},"filter": {"term": {"split": "valid"}}}}
 # the elasticsearch query string for matching calibration documents
-calibration.splitQuery={"bool": {"must": {"match_all": {}},"filter": {"term": {"split": "valid"}}}}
+calibration.splitQuery={"bool": {"must": {"match_all": {}},"filter": {"term": {"split": "cali"}}}}
 
 # generate training set
 createTrainSet=true

fix typo in config file