{"_dirty_type":"upd","class_fqid":"com-snaplogic-pipeline_8","snode_id":"59b9cbd395526e0560b2be71","instance_id":"7cf8ac56-5984-42ce-bcc8-48f8ac1e0687","instance_version":15,"link_map":{"link111":{"src_view_id":"output0","dst_view_id":"input0","src_id":"6a411cec-4c04-491c-afc9-8f3128f19839","dst_id":"8a7d59c4-bc73-4ccf-aed9-28040468bc04","isGoto":false},"link110":{"src_view_id":"output101","dst_view_id":"input101","src_id":"a51ad248-6781-4c19-9a39-0a896a5c59e5","dst_id":"febd38a9-e7d8-4230-88bc-a62f2de89d19","isGoto":false},"link113":{"src_view_id":"output0","dst_view_id":"input0","src_id":"9df7dc5d-d47e-4bde-acfe-160e4e925843","dst_id":"da868e91-d8ae-435c-ad56-da3e3520dd34","isGoto":false},"link112":{"src_view_id":"output0","dst_view_id":"input0","src_id":"febd38a9-e7d8-4230-88bc-a62f2de89d19","dst_id":"6a411cec-4c04-491c-afc9-8f3128f19839","isGoto":false},"link115":{"src_view_id":"error0","dst_view_id":"input0","src_id":"3185886e-05d0-4faf-845d-27e9ebfa4b2d","dst_id":"ee9cede8-6d30-4e45-a1ad-2ec1a3840f72","isGoto":false},"link114":{"src_view_id":"output101","dst_view_id":"input102","src_id":"da868e91-d8ae-435c-ad56-da3e3520dd34","dst_id":"a51ad248-6781-4c19-9a39-0a896a5c59e5","isGoto":false},"link106":{"src_view_id":"output0","dst_view_id":"input0","src_id":"ee9cede8-6d30-4e45-a1ad-2ec1a3840f72","dst_id":"9d44013e-00ea-483c-9c50-b8364ba1fc0b","isGoto":false}},"link_serial":116,"property_map":{"info":{"notes":{"value":null},"author":{"value":"psung@snaplogic.com"},"purpose":{"value":null},"pipeline_doc_uri":{"value":null},"label":{"value":"PySpark - wordcount"}},"input":{},"settings":{"param_table":{"value":[]},"imports":{"value":[]}},"error":{"a51ad248-6781-4c19-9a39-0a896a5c59e5_error0":{"view_type":{"value":"document"},"label":{"value":"PySpark - error0"}},"error_behavior":{"value":"none"}},"output":{"3185886e-05d0-4faf-845d-27e9ebfa4b2d_output101":{"view_type":{"value":"document"},"label":{"value":"PySpark - output0"}}}},"render_map":{"pan_x_num":0,"default_snaplex":"5854427995526e2f8debfc6f","scale_ratio":1,"detail_map":{"9d44013e-00ea-483c-9c50-b8364ba1fc0b":{"grid_x_int":4,"error":{},"grid_y_int":3,"rot_tail_int":0,"input":{},"output":{},"rot_int":0},"9df7dc5d-d47e-4bde-acfe-160e4e925843":{"grid_x_int":1,"rot_int":0,"grid_y_int":2,"rot_tail_int":0,"output":{}},"6a411cec-4c04-491c-afc9-8f3128f19839":{"grid_x_int":5,"grid_y_int":2,"rot_tail_int":0,"output":{},"input":{},"rot_int":0},"8a7d59c4-bc73-4ccf-aed9-28040468bc04":{"grid_x_int":6,"rot_int":0,"grid_y_int":2,"rot_tail_int":0,"input":{}},"da868e91-d8ae-435c-ad56-da3e3520dd34":{"grid_x_int":2,"rot_int":0,"grid_y_int":2,"rot_tail_int":0},"a51ad248-6781-4c19-9a39-0a896a5c59e5":{"grid_x_int":3,"rot_int":0,"grid_y_int":2,"rot_tail_int":0,"error":{"error0":{"dx_int":0,"rot_int":0,"dy_int":-1}}},"ee9cede8-6d30-4e45-a1ad-2ec1a3840f72":{"grid_x_int":3,"error":{},"grid_y_int":3,"rot_tail_int":0,"input":{},"output":{},"rot_int":0},"3185886e-05d0-4faf-845d-27e9ebfa4b2d":{"grid_x_int":2,"grid_y_int":4,"rot_tail_int":0,"error":{"error0":{"dx_int":0,"rot_int":0,"dy_int":-1}},"input":{},"rot_int":0},"febd38a9-e7d8-4230-88bc-a62f2de89d19":{"grid_x_int":4,"rot_int":0,"grid_y_int":2,"rot_tail_int":0}},"pan_y_num":0},"snap_map":{"9d44013e-00ea-483c-9c50-b8364ba1fc0b":{"class_fqid":"com-snaplogic-snaps-test-assertequals_1-c439","class_id":"com-snaplogic-snaps-test-assertequals","class_version":1,"instance_fqid":"9d44013e-00ea-483c-9c50-b8364ba1fc0b_1","instance_id":"9d44013e-00ea-483c-9c50-b8364ba1fc0b","instance_version":1,"property_map":{"info":{"notes":{"value":null},"label":{"value":"Assert Equals"}},"view_serial":100,"error":{"error0":{"label":{"value":"error0"},"view_type":{"value":"document"}},"error_behavior":{"value":"fail"}},"input":{"input0":{"label":{"value":"input0"},"view_type":{"value":"document"}}},"settings":{"editable_content":{"value":"## Enter your expected JSON-encoded data in this space.\n## \"[]\" is required, i.e. should be a List of objects.\n\n[\n  {\n    \"error\": \"PySpark script failed\",\n    \"reason\": \"exit code = 1\",\n    \"resolution\": \"Please address reported issue.\",\n    \"spark-command\": true,\n    \"FileNotFound\": true\n  }\n]\n\n\n## Tips:\n##  * The sample data above will generate a single empty document, uncomment\n##    the line in the middle to include the sample fields.  Adding more\n##    objects to the root array will cause the snap to generate more\n##    than one document."}}},"class_build_tag":"c439"},"9df7dc5d-d47e-4bde-acfe-160e4e925843":{"class_fqid":"com-snaplogic-snaps-binary-simpleread_1-MULTIPLE3924","class_id":"com-snaplogic-snaps-binary-simpleread","class_version":1,"instance_fqid":"9df7dc5d-d47e-4bde-acfe-160e4e925843_1","instance_id":"9df7dc5d-d47e-4bde-acfe-160e4e925843","instance_version":1,"property_map":{"info":{"notes":{"value":null},"label":{"value":"File Reader"}},"view_serial":100,"account":{"account_ref":{"expression":false,"value":{}}},"settings":{"executeDuringPreview":{"value":true},"preventURLEncoding":{"value":false},"filePath":{"expression":false,"value":"sample.txt"}},"error":{"error0":{"view_type":{"value":"document"},"label":{"value":"error0"}},"error_behavior":{"value":"fail"}},"output":{"output0":{"view_type":{"value":"binary"},"label":{"value":"output0"}}}},"class_build_tag":"MULTIPLE3924"},"6a411cec-4c04-491c-afc9-8f3128f19839":{"class_fqid":"com-snaplogic-snaps-transform-csvparser_3-c439","class_id":"com-snaplogic-snaps-transform-csvparser","class_version":3,"instance_fqid":"6a411cec-4c04-491c-afc9-8f3128f19839_1","instance_id":"6a411cec-4c04-491c-afc9-8f3128f19839","instance_version":1,"property_map":{"info":{"label":{"value":"CSV Parser"}},"view_serial":100,"input":{"input0":{"view_type":{"value":"binary"},"label":{"value":"input0"}}},"settings":{"escapeChar":{"value":"\\"},"quoteCharacter":{"value":"\""},"containsHeader":{"value":true},"charset":{"value":"Auto BOM detect"},"skipLines":{"value":0},"delimiter":{"value":","},"errorPolicy":{"value":"Both"},"validateHeaders":{"value":false},"ignoreEmptyData":{"value":true}},"error":{"error0":{"view_type":{"value":"document"},"label":{"value":"error0"}},"error_behavior":{"value":"fail"}},"output":{"output0":{"view_type":{"value":"document"},"label":{"value":"output0"}}}},"class_build_tag":"c439"},"8a7d59c4-bc73-4ccf-aed9-28040468bc04":{"class_fqid":"com-snaplogic-snaps-test-assertequals_1-c439","class_id":"com-snaplogic-snaps-test-assertequals","class_version":1,"instance_fqid":"8a7d59c4-bc73-4ccf-aed9-28040468bc04_1","instance_id":"8a7d59c4-bc73-4ccf-aed9-28040468bc04","instance_version":1,"property_map":{"info":{"notes":{"value":null},"label":{"value":"Assert Equals"}},"view_serial":100,"error":{"error0":{"label":{"value":"error0"},"view_type":{"value":"document"}},"error_behavior":{"value":"fail"}},"input":{"input0":{"label":{"value":"input0"},"view_type":{"value":"document"}}},"settings":{"editable_content":{"value":"## Enter your expected JSON-encoded data in this space.\n## \"[]\" is required, i.e. should be a List of objects.\n\n[\n  {\n    \"word\": \"joy\",\n    \"count\": \"1\"\n  },\n  {\n    \"word\": \"love\",\n    \"count\": \"3\"\n  },\n  {\n    \"word\": \"happy\",\n    \"count\": \"2\"\n  }\n]\n\n\n## Tips:\n##  * The sample data above will generate a single empty document, uncomment\n##    the line in the middle to include the sample fields.  Adding more\n##    objects to the root array will cause the snap to generate more\n##    than one document."}}},"class_build_tag":"c439"},"da868e91-d8ae-435c-ad56-da3e3520dd34":{"class_fqid":"com-snaplogic-snaps-hadoop-hdfswrite_1-hadoop3921","class_id":"com-snaplogic-snaps-hadoop-hdfswrite","class_version":1,"instance_fqid":"da868e91-d8ae-435c-ad56-da3e3520dd34_3","instance_id":"da868e91-d8ae-435c-ad56-da3e3520dd34","instance_version":3,"property_map":{"info":{"notes":{"value":null},"label":{"value":"HDFS Writer"}},"view_serial":101,"account":{"account_ref":{"expression":false,"value":{}}},"input":{"input0":{"view_type":{"value":"binary"},"label":{"value":"input0"}}},"settings":{"UserPermissionsKey":{"value":[]},"executable_during_suggest":{"value":true},"filter":{"value":"*"},"UserImpersonation":{"value":false},"file":{"expression":false,"value":"sample.txt"},"directory":{"expression":false,"value":"hdfs://sl-cdh5-5-qabd-01.clouddev.snaplogic.com:8020/tmp/"},"fileAction":{"value":"OVERWRITE"}},"error":{"error0":{"view_type":{"value":"document"},"label":{"value":"error0"}},"error_behavior":{"value":"fail"}},"output":{"output101":{"label":{"value":"output0"},"view_type":{"value":"document"}}}},"class_build_tag":"hadoop3921"},"a51ad248-6781-4c19-9a39-0a896a5c59e5":{"class_fqid":"com-snaplogic-snaps-script-pyspark_1-SNAPSHOT","class_id":"com-snaplogic-snaps-script-pyspark","class_version":1,"instance_fqid":"a51ad248-6781-4c19-9a39-0a896a5c59e5_3","instance_id":"a51ad248-6781-4c19-9a39-0a896a5c59e5","instance_version":3,"property_map":{"info":{"notes":{"value":null},"label":{"value":"PySpark"}},"view_serial":102,"input":{"input102":{"label":{"value":"input0"},"view_type":{"value":"document"}}},"settings":{"scriptArgs":{"expression":false,"value":"hdfs:///tmp/sample.txt\nhdfs:///tmp/output.dir/"},"sparkSubmitArgs":{"expression":false,"value":"--packages com.databricks:spark-csv_2.10:1.5.0"},"editable_content":{"value":"### PySpark Word Count script template ###\n################################################################\n# If you are not familiar with Spark Programming,              #\n# please check Spark programming guide for more detail.        #\n# http://spark.apache.org/docs/latest/programming-guide.html   #\n################################################################\n\nimport sys\nfrom operator import add\nfrom pyspark.sql import SQLContext\nfrom pyspark import SparkContext\n\nif __name__ == \"__main__\":\n    # check if the number of arguments is 2\n    if len(sys.argv) < 3:\n        print >> sys.stderr, \"Usage: wordcount <input file> <output file>\"\n        exit(-1)\n    # get contexts\n    sc = SparkContext()\n    sqlContext = SQLContext(sc)\n    # do word-count\n    lines = sc.textFile(sys.argv[1], 1)\n    counts = lines.flatMap(lambda x: x.split(' ')) \\\n                  .map(lambda x: (x, 1)) \\\n                  .reduceByKey(add)\n    output = counts.collect()\n    # define column names for CSV output\n    df = sqlContext.createDataFrame(counts)\n    oldColumns = df.schema.names\n    newColumns = [\"word\", \"count\"]\n    newdf = reduce(lambda df, idx: df.withColumnRenamed(oldColumns[idx], newColumns[idx]), xrange(len(oldColumns)), df)\n    # write wordcount result to output file with overwrite mode\n    newdf.write.mode(\"overwrite\").format(\"com.databricks.spark.csv\").option(\"header\", \"true\").save(sys.argv[2])\n    # print wordcount result to standard output\n    print \"word, count\"\n    for (word, count) in output:\n        print \"%s, %i\" % (word, count)"},"sparkHome":{"expression":false,"value":"/opt/cloudera/parcels/CDH-5.8.4-1.cdh5.8.4.p0.5/lib/spark"},"executable_during_suggest":{"value":true}},"error":{"error0":{"label":{"value":"error0"},"view_type":{"value":"document"}},"error_behavior":{"value":"continue"}},"output":{"output101":{"view_type":{"value":"document"},"label":{"value":"output0"}}}},"class_build_tag":"SNAPSHOT"},"ee9cede8-6d30-4e45-a1ad-2ec1a3840f72":{"class_fqid":"com-snaplogic-snaps-transform-datatransform_4-c439","class_id":"com-snaplogic-snaps-transform-datatransform","class_version":4,"instance_fqid":"ee9cede8-6d30-4e45-a1ad-2ec1a3840f72_1","instance_id":"ee9cede8-6d30-4e45-a1ad-2ec1a3840f72","instance_version":1,"property_map":{"info":{"notes":{"value":null},"label":{"value":"Mapper"}},"view_serial":100,"input":{"input0":{"view_type":{"value":"document"},"label":{"value":"input0"}}},"settings":{"nullSafeAccess":{"value":false},"passThrough":{"value":false},"transformations":{"value":{"mappingTable":{"value":[{"expression":{"expression":true,"value":"$error"},"targetPath":{"value":"$error"}},{"expression":{"expression":true,"value":"$reason"},"targetPath":{"value":"$reason"}},{"expression":{"expression":true,"value":"$resolution"},"targetPath":{"value":"$resolution"}},{"expression":{"expression":true,"value":"$['spark-submit command'].contains(\"/bin/spark-submit\")"},"targetPath":{"value":"$spark-command"}},{"expression":{"expression":true,"value":"$['standard output'].contains(\"Input path does not exist\")"},"targetPath":{"value":"$FileNotFound"}}]},"mappingRoot":{"value":"$"}}}},"error":{"error0":{"view_type":{"value":"document"},"label":{"value":"error0"}},"error_behavior":{"value":"fail"}},"output":{"output0":{"view_type":{"value":"document"},"label":{"value":"output0"}}}},"class_build_tag":"c439"},"3185886e-05d0-4faf-845d-27e9ebfa4b2d":{"class_fqid":"com-snaplogic-snaps-script-pyspark_1-SNAPSHOT","class_id":"com-snaplogic-snaps-script-pyspark","class_version":1,"instance_fqid":"3185886e-05d0-4faf-845d-27e9ebfa4b2d_1","instance_id":"3185886e-05d0-4faf-845d-27e9ebfa4b2d","instance_version":1,"property_map":{"info":{"notes":{"value":null},"label":{"value":"PySpark"}},"view_serial":102,"input":{},"settings":{"scriptArgs":{"expression":false,"value":"hdfs:///tmp/missing.txt\nhdfs:///tmp/output.dir/"},"sparkHome":{"expression":false,"value":"/opt/cloudera/parcels/CDH-5.8.4-1.cdh5.8.4.p0.5/lib/spark"},"editable_content":{"value":"### PySpark Word Count script template ###\n################################################################\n# If you are not familiar with Spark Programming,              #\n# please check Spark programming guide for more detail.        #\n# http://spark.apache.org/docs/latest/programming-guide.html   #\n################################################################\n\nimport sys\nfrom operator import add\nfrom pyspark.sql import SQLContext\nfrom pyspark import SparkContext\n\nif __name__ == \"__main__\":\n    # check if the number of arguments is 2\n    if len(sys.argv) < 3:\n        print >> sys.stderr, \"Usage: wordcount <input file> <output file>\"\n        exit(-1)\n    # get contexts\n    sc = SparkContext()\n    sqlContext = SQLContext(sc)\n    # do word-count\n    lines = sc.textFile(sys.argv[1], 1)\n    counts = lines.flatMap(lambda x: x.split(' ')) \\\n                  .map(lambda x: (x, 1)) \\\n                  .reduceByKey(add)\n    output = counts.collect()\n    # define column names for CSV output\n    df = sqlContext.createDataFrame(counts)\n    oldColumns = df.schema.names\n    newColumns = [\"word\", \"count\"]\n    newdf = reduce(lambda df, idx: df.withColumnRenamed(oldColumns[idx], newColumns[idx]), xrange(len(oldColumns)), df)\n    # write wordcount result to output file with overwrite mode\n    newdf.write.mode(\"overwrite\").format(\"com.databricks.spark.csv\").option(\"header\", \"true\").save(sys.argv[2])\n    # print wordcount result to standard output\n    print \"word, count\"\n    for (word, count) in output:\n        print \"%s, %i\" % (word, count)"},"sparkSubmitArgs":{"expression":false,"value":"--packages com.databricks:spark-csv_2.10:1.5.0"},"executable_during_suggest":{"value":true}},"error":{"error0":{"view_type":{"value":"document"},"label":{"value":"error0"}},"error_behavior":{"value":"continue"}},"output":{"output101":{"label":{"value":"output0"},"view_type":{"value":"document"}}}},"class_build_tag":"SNAPSHOT"},"febd38a9-e7d8-4230-88bc-a62f2de89d19":{"class_fqid":"com-snaplogic-snaps-hadoop-hdfsread_1-hadoop3921","class_id":"com-snaplogic-snaps-hadoop-hdfsread","class_version":1,"instance_fqid":"febd38a9-e7d8-4230-88bc-a62f2de89d19_5","instance_id":"febd38a9-e7d8-4230-88bc-a62f2de89d19","instance_version":5,"property_map":{"info":{"notes":{"value":null},"label":{"value":"HDFS Reader"}},"view_serial":101,"account":{"account_ref":{"expression":false,"value":{}}},"settings":{"directory":{"expression":false,"value":"hdfs://sl-cdh5-5-qabd-01.clouddev.snaplogic.com:8020/tmp/output.dir"},"filter":{"value":"part*"},"UserImpersonation":{"value":false},"file":{"expression":false,"value":null}},"output":{"output0":{"view_type":{"value":"binary"},"label":{"value":"output0"}}},"error":{"error0":{"view_type":{"value":"document"},"label":{"value":"error0"}},"error_behavior":{"value":"fail"}},"input":{"input101":{"label":{"value":"input0"},"view_type":{"value":"document"}}}},"class_build_tag":"hadoop3921"}},"path_id":"/QA/shared","path_snode":"5566505895526e3fc7775b03"}