We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
I specified below for index task:
{ "type" : "index_hadoop", "id" : "index_hadoop_dictweb.pageview.aged_2017-04-05T11:29:09.308+08:00", "spec" : { "dataSchema" : { "dataSource" : "dictweb.pageview.aged", "parser" : { "type" : "string", "parseSpec" : { "format" : "json", "timestampSpec" : { "column" : "ts", "format" : "millis" }, "dimensionsSpec" : { "dimensions" : [ "_web.remote.result.typo", "Browser", "Platform", "_nref", "_nref-query", "_nrefDomain", "_nrefHost", "_nrefPath", "_nrefUrl", "accessCode", "basePath", "device", "has_stopRefer", "ip", "ipCom", "ipCountry", "ipLocation", "ipProvince", "isNetEaseOfficeIp", "keyfrom", "notCare", "pos", "referer-query", "refererDomain", "refererUrl", "vendor", "visitKey", "visitKeyDesc", "intDay", "_ncoo_isSessionStart", "_ncoo_isDayUidStart", "_ncoo_firstVisitDay", "_ncoo_firstVisitDayIn180Day", "_ncoo_firstVisit_vendor", "_ncoo_firstVisit_keyfrom", "has_uid__ncoo", "_ncoo_dailyReportOldUser", "_ncoo_ne_dailyReportOldUser", "_nssn_isSessionStart", "_nssn_isDayUidStart", "_nssn_firstVisitDay", "_nssn_firstVisitDayIn180Day", "_nssn_firstVisit_vendor", "_nssn_firstVisit_keyfrom", "has_uid__nssn", "_nssn_dailyReportOldUser", "_nssn_ne_dailyReportOldUser" ], "spatialDimensions" : [ ] } } }, "metricsSpec" : [ { "type" : "count", "name" : "cnt" }, { "type" : "thetaSketch", "name" : "_ncoo_sessionId_sketch_x256", "fieldName" : "_ncoo_sessionId_Dup", "size" : 4194304, "shouldFinalize" : true, "isInputThetaSketch" : false, "errorBoundsStdDev" : null }, { "type" : "thetaSketch", "name" : "_ncoo_sketch", "fieldName" : "_ncoo_Dup", "size" : 16384, "shouldFinalize" : true, "isInputThetaSketch" : false, "errorBoundsStdDev" : null }, { "type" : "thetaSketch", "name" : "_ncoo_sketch_x256", "fieldName" : "_ncoo_Dup", "size" : 4194304, "shouldFinalize" : true, "isInputThetaSketch" : false, "errorBoundsStdDev" : null }, { "type" : "longSum", "name" : "_ncoo_isSessionStart_cnt", "fieldName" : "_ncoo_isSessionStart_Dup", "expression" : null }, { "type" : "longSum", "name" : "_ncoo_sessionActionCountOne", "fieldName" : "_ncoo_sessionActionCountOne", "expression" : null }, { "type" : "longSum", "name" : "_ncoo_sessionStaySecsCnt", "fieldName" : "_ncoo_sessionStaySecs", "expression" : null }, { "type" : "longSum", "name" : "_ncoo_sessionActionCount", "fieldName" : "_ncoo_sessionActionCount", "expression" : null }, { "type" : "longSum", "name" : "_ncoo_isDayUidStart_cnt", "fieldName" : "_ncoo_isDayUidStart_Dup", "expression" : null }, { "type" : "approxHistogramFold", "name" : "_ncoo_sessionStaySecsHistFold", "fieldName" : "_ncoo_sessionStaySecs", "resolution" : 100, "numBuckets" : 7, "lowerLimit" : 0.0, "upperLimit" : 60000.0 }, { "type" : "thetaSketch", "name" : "_nssn_sessionId_sketch_x256", "fieldName" : "_nssn_sessionId_Dup", "size" : 4194304, "shouldFinalize" : true, "isInputThetaSketch" : false, "errorBoundsStdDev" : null }, { "type" : "thetaSketch", "name" : "_nssn_sketch", "fieldName" : "_nssn_Dup", "size" : 16384, "shouldFinalize" : true, "isInputThetaSketch" : false, "errorBoundsStdDev" : null }, { "type" : "thetaSketch", "name" : "_nssn_sketch_x256", "fieldName" : "_nssn_Dup", "size" : 4194304, "shouldFinalize" : true, "isInputThetaSketch" : false, "errorBoundsStdDev" : null }, { "type" : "longSum", "name" : "_nssn_isSessionStart_cnt", "fieldName" : "_nssn_isSessionStart_Dup", "expression" : null }, { "type" : "longSum", "name" : "_nssn_sessionActionCountOne", "fieldName" : "_nssn_sessionActionCountOne", "expression" : null }, { "type" : "longSum", "name" : "_nssn_sessionStaySecsCnt", "fieldName" : "_nssn_sessionStaySecs", "expression" : null }, { "type" : "longSum", "name" : "_nssn_sessionActionCount", "fieldName" : "_nssn_sessionActionCount", "expression" : null }, { "type" : "longSum", "name" : "_nssn_isDayUidStart_cnt", "fieldName" : "_nssn_isDayUidStart_Dup", "expression" : null }, { "type" : "approxHistogramFold", "name" : "_nssn_sessionStaySecsHistFold", "fieldName" : "_nssn_sessionStaySecs", "resolution" : 100, "numBuckets" : 7, "lowerLimit" : 0.0, "upperLimit" : 60000.0 }, { "type" : "approxHistogramFold", "name" : "hist_fold_stopRefer", "fieldName" : "stopRefer", "resolution" : 512, "numBuckets" : 7, "lowerLimit" : 0.0, "upperLimit" : 60000.0 } ], "granularitySpec" : { "type" : "uniform", "segmentGranularity" : "DAY", "queryGranularity" : "DAY", "rollup" : true, "intervals" : [ "2016-03-29T00:00:00.000+08:00/2016-03-30T00:00:00.000+08:00" ] } }, "ioConfig" : { "type" : "hadoop", "inputSpec" : { "type" : "static", "paths" : "/exec/analysis/exp/dictweb.pageview.aged-kv-json/20160329" }, "metadataUpdateSpec" : null, "segmentOutputPath" : null }, "tuningConfig" : { "type" : "hadoop", "workingPath" : null, "version" : "2017-04-05T11:29:09.308+08:00", "partitionsSpec" : { "type" : "hashed", "targetPartitionSize" : 5000000, "maxPartitionSize" : 7500000, "assumeGrouped" : false, "numShards" : -1, "partitionDimensions" : [ ] }, "shardSpecs" : { }, "indexSpec" : { "bitmap" : { "type" : "concise" }, "dimensionCompression" : "lz4", "metricCompression" : "lz4", "longEncoding" : "longs" }, "maxRowsInMemory" : 10000, "leaveIntermediate" : false, "cleanupOnFailure" : true, "overwriteFiles" : false, "ignoreInvalidRows" : true, "jobProperties" : { "mapred.task.timeout" : "7200000", "mapreduce.job.queuename" : "default", "mapreduce.map.speculative" : "true", "mapreduce.reduce.speculative" : "false", "mapreduce.job.running.map.limit" : "128", "mapreduce.job.running.reduce.limit" : "128", "mapreduce.job.user.classpath.first" : "false", "mapreduce.job.reduce.slowstart.completedmaps" : "1.0", "mapreduce.job.classloader" : "true", "mapreduce.job.classloader.system.classes" : "java.,-javax.validation.,javax.,org.apache.log4j.,org.apache.hadoop.,org.apache.xerces.,org.w3c.dom.,org.xml.", "mapreduce.reduce.shuffle.input.buffer.percent" : "0.3", "mapreduce.input.fileinputformat.split.minsize" : "134217728", "mapreduce.input.fileinputformat.split.maxsize" : "134217728", "mapreduce.map.memory.mb" : "3000", "mapreduce.task.io.sort.mb" : "256", "mapreduce.task.io.sort.factor" : "100", "mapreduce.map.output.compress" : "true", "yarn.app.mapreduce.am.command-opts" : "-Xmx2048m -XX:ErrorFile=/tmp/hs_err_task_@taskid@_pid%p.log", "mapreduce.map.java.opts" : "-XX:+UseParallelOldGC -XX:ParallelGCThreads=4 -XX:-OmitStackTraceInFastThrow -server -Xmn400m -Xms2200m -Xmx2200m -Duser.timezone=Asia/Shanghai -Dfile.encoding=UTF-8 -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:PermSize=256m -XX:MaxPermSize=256m -XX:-UseGCOverheadLimit -XX:ErrorFile=/tmp/hs_err_task_@taskid@_pid%p.log", "mapreduce.reduce.memory.mb" : "8001", "mapreduce.reduce.java.opts" : "-XX:-OmitStackTraceInFastThrow -server -XX:+UseG1GC -Xmx3801m -Xms3801m -Duser.timezone=Asia/Shanghai -Dfile.encoding=UTF-8 -XX:-DisableExplicitGC -XX:MaxDirectMemorySize=2230m -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:-UseGCOverheadLimit -XX:ErrorFile=/tmp/hs_err_task_@taskid@_pid%p.log" }, "combineText" : true, "useCombiner" : true, "buildV9Directly" : true, "numBackgroundPersistThreads" : 1, "forceExtendableShardSpecs" : false, "useExplicitVersion" : false }, "uniqueId" : "d9d9fb2e3a354fe2810c100ddb336ade" }, "hadoopDependencyCoordinates" : null, "classpathPrefix" : null, "context" : { "druid.indexer.runner.javaOpts" : "-XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:MaxPermSize=256m -XX:PermSize=256m -Xms1024m -Xmx1024m -XX:MaxDirectMemorySize=1220m" }, "groupId" : "index_hadoop_dictweb.pageview.aged_2017-04-05T11:29:09.308+08:00", "dataSource" : "dictweb.pageview.aged", "resource" : { "availabilityGroup" : "index_hadoop_dictweb.pageview.aged_2017-04-05T11:29:09.308+08:00", "requiredCapacity" : 1 } }
In ingest spec, I specified:
"intervals" : [ "2016-03-29T00:00:00.000+08:00/2016-03-30T00:00:00.000+08:00" ]
but generated segments:
20160329T080000.000+0800_20160330T080000.000+0800 20160328T080000.000+0800_20160329T080000.000+0800
task log
2017-04-05T11:55:48,701 INFO [task-runner-0-priority-0] io.druid.indexer.IndexGeneratorJob - Adding segment dictweb.pageview.aged_2016-03-28T08:00:00.000+08:00_2016-03-29T08:00:00.000+08:00_2017-04-05T11:29:09.311+08:00 to the list of published segments 2017-04-05T11:55:48,703 INFO [task-runner-0-priority-0] io.druid.indexer.IndexGeneratorJob - Adding segment dictweb.pageview.aged_2016-03-29T08:00:00.000+08:00_2016-03-30T08:00:00.000+08:00_2017-04-05T11:29:09.311+08:00 to the list of published segments 2017-04-05T11:55:48,703 INFO [task-runner-0-priority-0] io.druid.indexer.JobHelper - Deleting path[/exec/analysis/tmp/dictweb.pageview.aged/2017-04-05T112909.311+0800_d9d9fb2e3a354fe2810c100ddb336ade] 2017-04-05T11:55:48,757 INFO [task-runner-0-priority-0] io.druid.indexing.common.actions.RemoteTaskActionClient - Performing action for task[index_hadoop_dictweb.pageview.aged_2017-04-05T11:29:09.308+08:00]: SegmentInsertAction{segments=[DataSegment{size=65791532, shardSpec=NoneShardSpec, metrics=[cnt, _ncoo_sessionId_sketch_x256, _ncoo_sketch, _ncoo_sketch_x256, _ncoo_isSessionStart_cnt, _ncoo_sessionActionCountOne, _ncoo_sessionStaySecsCnt, _ncoo_sessionActionCount, _ncoo_isDayUidStart_cnt, _ncoo_sessionStaySecsHistFold, _nssn_sessionId_sketch_x256, _nssn_sketch, _nssn_sketch_x256, _nssn_isSessionStart_cnt, _nssn_sessionActionCountOne, _nssn_sessionStaySecsCnt, _nssn_sessionActionCount, _nssn_isDayUidStart_cnt, _nssn_sessionStaySecsHistFold, hist_fold_stopRefer], dimensions=[_web.remote.result.typo, Browser, Platform, _nref, _nref-query, _nrefDomain, _nrefHost, _nrefPath, _nrefUrl, accessCode, basePath, device, has_stopRefer, ip, ipCom, ipCountry, ipLocation, ipProvince, isNetEaseOfficeIp, keyfrom, notCare, pos, referer-query, refererDomain, refererUrl, vendor, visitKey, visitKeyDesc, intDay, _ncoo_isSessionStart, _ncoo_isDayUidStart, _ncoo_firstVisitDay, _ncoo_firstVisitDayIn180Day, _ncoo_firstVisit_vendor, _ncoo_firstVisit_keyfrom, has_uid__ncoo, _ncoo_dailyReportOldUser, _ncoo_ne_dailyReportOldUser, _nssn_isSessionStart, _nssn_isDayUidStart, _nssn_firstVisitDay, _nssn_firstVisitDayIn180Day, _nssn_firstVisit_vendor, _nssn_firstVisit_keyfrom, has_uid__nssn, _nssn_dailyReportOldUser, _nssn_ne_dailyReportOldUser], version='2017-04-05T11:29:09.311+08:00', loadSpec={type=hdfs, path=hdfs://eadhadoop/exec/analysis/druid/deepStorage_0_8_3/dictweb.pageview.aged/20160328T080000.000+0800_20160329T080000.000+0800/2017-04-05T11_29_09.311+08_00/0_index.zip}, interval=2016-03-28T08:00:00.000+08:00/2016-03-29T08:00:00.000+08:00, dataSource='dictweb.pageview.aged', binaryVersion='9'}]} 2017-04-05T11:55:48,763 INFO [task-runner-0-priority-0] io.druid.indexing.common.actions.RemoteTaskActionClient - Submitting action for task[index_hadoop_dictweb.pageview.aged_2017-04-05T11:29:09.308+08:00] to overlord[http://hd020.corp.yodao.com:8195/druid/indexer/v1/action]: SegmentInsertAction{segments=[DataSegment{size=65791532, shardSpec=NoneShardSpec, metrics=[cnt, _ncoo_sessionId_sketch_x256, _ncoo_sketch, _ncoo_sketch_x256, _ncoo_isSessionStart_cnt, _ncoo_sessionActionCountOne, _ncoo_sessionStaySecsCnt, _ncoo_sessionActionCount, _ncoo_isDayUidStart_cnt, _ncoo_sessionStaySecsHistFold, _nssn_sessionId_sketch_x256, _nssn_sketch, _nssn_sketch_x256, _nssn_isSessionStart_cnt, _nssn_sessionActionCountOne, _nssn_sessionStaySecsCnt, _nssn_sessionActionCount, _nssn_isDayUidStart_cnt, _nssn_sessionStaySecsHistFold, hist_fold_stopRefer], dimensions=[_web.remote.result.typo, Browser, Platform, _nref, _nref-query, _nrefDomain, _nrefHost, _nrefPath, _nrefUrl, accessCode, basePath, device, has_stopRefer, ip, ipCom, ipCountry, ipLocation, ipProvince, isNetEaseOfficeIp, keyfrom, notCare, pos, referer-query, refererDomain, refererUrl, vendor, visitKey, visitKeyDesc, intDay, _ncoo_isSessionStart, _ncoo_isDayUidStart, _ncoo_firstVisitDay, _ncoo_firstVisitDayIn180Day, _ncoo_firstVisit_vendor, _ncoo_firstVisit_keyfrom, has_uid__ncoo, _ncoo_dailyReportOldUser, _ncoo_ne_dailyReportOldUser, _nssn_isSessionStart, _nssn_isDayUidStart, _nssn_firstVisitDay, _nssn_firstVisitDayIn180Day, _nssn_firstVisit_vendor, _nssn_firstVisit_keyfrom, has_uid__nssn, _nssn_dailyReportOldUser, _nssn_ne_dailyReportOldUser], version='2017-04-05T11:29:09.311+08:00', loadSpec={type=hdfs, path=hdfs://eadhadoop/exec/analysis/druid/deepStorage_0_8_3/dictweb.pageview.aged/20160328T080000.000+0800_20160329T080000.000+0800/2017-04-05T11_29_09.311+08_00/0_index.zip}, interval=2016-03-28T08:00:00.000+08:00/2016-03-29T08:00:00.000+08:00, dataSource='dictweb.pageview.aged', binaryVersion='9'}]} 2017-04-05T11:55:48,763 INFO [task-runner-0-priority-0] com.metamx.http.client.pool.ChannelResourceFactory - Generating: http://hd020.corp.yodao.com:8195 2017-04-05T11:55:48,773 INFO [task-runner-0-priority-0] io.druid.indexing.common.actions.RemoteTaskActionClient - Performing action for task[index_hadoop_dictweb.pageview.aged_2017-04-05T11:29:09.308+08:00]: SegmentInsertAction{segments=[DataSegment{size=822467758, shardSpec=NoneShardSpec, metrics=[cnt, _ncoo_sessionId_sketch_x256, _ncoo_sketch, _ncoo_sketch_x256, _ncoo_isSessionStart_cnt, _ncoo_sessionActionCountOne, _ncoo_sessionStaySecsCnt, _ncoo_sessionActionCount, _ncoo_isDayUidStart_cnt, _ncoo_sessionStaySecsHistFold, _nssn_sessionId_sketch_x256, _nssn_sketch, _nssn_sketch_x256, _nssn_isSessionStart_cnt, _nssn_sessionActionCountOne, _nssn_sessionStaySecsCnt, _nssn_sessionActionCount, _nssn_isDayUidStart_cnt, _nssn_sessionStaySecsHistFold, hist_fold_stopRefer], dimensions=[_web.remote.result.typo, Browser, Platform, _nref, _nref-query, _nrefDomain, _nrefHost, _nrefPath, _nrefUrl, accessCode, basePath, device, has_stopRefer, ip, ipCom, ipCountry, ipLocation, ipProvince, isNetEaseOfficeIp, keyfrom, notCare, pos, referer-query, refererDomain, refererUrl, vendor, visitKey, visitKeyDesc, intDay, _ncoo_isSessionStart, _ncoo_isDayUidStart, _ncoo_firstVisitDay, _ncoo_firstVisitDayIn180Day, _ncoo_firstVisit_vendor, _ncoo_firstVisit_keyfrom, has_uid__ncoo, _ncoo_dailyReportOldUser, _ncoo_ne_dailyReportOldUser, _nssn_isSessionStart, _nssn_isDayUidStart, _nssn_firstVisitDay, _nssn_firstVisitDayIn180Day, _nssn_firstVisit_vendor, _nssn_firstVisit_keyfrom, has_uid__nssn, _nssn_dailyReportOldUser, _nssn_ne_dailyReportOldUser], version='2017-04-05T11:29:09.311+08:00', loadSpec={type=hdfs, path=hdfs://eadhadoop/exec/analysis/druid/deepStorage_0_8_3/dictweb.pageview.aged/20160329T080000.000+0800_20160330T080000.000+0800/2017-04-05T11_29_09.311+08_00/0_index.zip}, interval=2016-03-29T08:00:00.000+08:00/2016-03-30T08:00:00.000+08:00, dataSource='dictweb.pageview.aged', binaryVersion='9'}]} 2017-04-05T11:55:48,774 INFO [task-runner-0-priority-0] io.druid.indexing.common.actions.RemoteTaskActionClient - Submitting action for task[index_hadoop_dictweb.pageview.aged_2017-04-05T11:29:09.308+08:00] to overlord[http://hd020.corp.yodao.com:8195/druid/indexer/v1/action]: SegmentInsertAction{segments=[DataSegment{size=822467758, shardSpec=NoneShardSpec, metrics=[cnt, _ncoo_sessionId_sketch_x256, _ncoo_sketch, _ncoo_sketch_x256, _ncoo_isSessionStart_cnt, _ncoo_sessionActionCountOne, _ncoo_sessionStaySecsCnt, _ncoo_sessionActionCount, _ncoo_isDayUidStart_cnt, _ncoo_sessionStaySecsHistFold, _nssn_sessionId_sketch_x256, _nssn_sketch, _nssn_sketch_x256, _nssn_isSessionStart_cnt, _nssn_sessionActionCountOne, _nssn_sessionStaySecsCnt, _nssn_sessionActionCount, _nssn_isDayUidStart_cnt, _nssn_sessionStaySecsHistFold, hist_fold_stopRefer], dimensions=[_web.remote.result.typo, Browser, Platform, _nref, _nref-query, _nrefDomain, _nrefHost, _nrefPath, _nrefUrl, accessCode, basePath, device, has_stopRefer, ip, ipCom, ipCountry, ipLocation, ipProvince, isNetEaseOfficeIp, keyfrom, notCare, pos, referer-query, refererDomain, refererUrl, vendor, visitKey, visitKeyDesc, intDay, _ncoo_isSessionStart, _ncoo_isDayUidStart, _ncoo_firstVisitDay, _ncoo_firstVisitDayIn180Day, _ncoo_firstVisit_vendor, _ncoo_firstVisit_keyfrom, has_uid__ncoo, _ncoo_dailyReportOldUser, _ncoo_ne_dailyReportOldUser, _nssn_isSessionStart, _nssn_isDayUidStart, _nssn_firstVisitDay, _nssn_firstVisitDayIn180Day, _nssn_firstVisit_vendor, _nssn_firstVisit_keyfrom, has_uid__nssn, _nssn_dailyReportOldUser, _nssn_ne_dailyReportOldUser], version='2017-04-05T11:29:09.311+08:00', loadSpec={type=hdfs, path=hdfs://eadhadoop/exec/analysis/druid/deepStorage_0_8_3/dictweb.pageview.aged/20160329T080000.000+0800_20160330T080000.000+0800/2017-04-05T11_29_09.311+08_00/0_index.zip}, interval=2016-03-29T08:00:00.000+08:00/2016-03-30T08:00:00.000+08:00, dataSource='dictweb.pageview.aged', binaryVersion='9'}]} 2017-04-05T11:55:48,774 INFO [task-runner-0-priority-0] com.metamx.http.client.pool.ChannelResourceFactory - Generating: http://hd020.corp.yodao.com:8195 2017-04-05T11:55:48,781 INFO [task-runner-0-priority-0] io.druid.indexing.overlord.TaskRunnerUtils - Task [index_hadoop_dictweb.pageview.aged_2017-04-05T11:29:09.308+08:00] status changed to [SUCCESS]. 2017-04-05T11:55:48,784 INFO [task-runner-0-priority-0] io.druid.indexing.worker.executor.ExecutorLifecycle - Task completed with status: { "id" : "index_hadoop_dictweb.pageview.aged_2017-04-05T11:29:09.308+08:00", "status" : "SUCCESS", "duration" : 1587705 }
The text was updated successfully, but these errors were encountered:
I found it is related to segmentGranularity setting. if I specified
"segmentGranularity" : { "type" : "period", "period" : "P1D", "timeZone" : "+08:00", "origin" : null }
other than
"segmentGranularity" : "DAY",
this "bug" do not appear.
Sorry, something went wrong.
No branches or pull requests
I specified below for index task:
In ingest spec, I specified:
but generated segments:
task log
The text was updated successfully, but these errors were encountered: