Schema Info

Schema Name: sfdl_prod_drivethru.websites_data_analytics_demo
  • created: 2025-02-28 14:05:11
  • disabled: False
  • format: raw
  • id: 3476
  • is_partitioned: True
  • is_schema_datatyped: True
  • name: 
    {
    "database": "sfdl_prod_drivethru",
    "table": "websites_data_analytics_demo"
}
  • type: drivethrujson
Schema Versions: 3
    • created: 2025-08-04 02:05:14
    • ddl: 
      [
    "CREATE DATABASE IF NOT EXISTS `sfdl_prod_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_prod_drivethru.websites_data_analytics_demo`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_prod_drivethru.websites_data_analytics_demo`\n(\n`siteId`STRING,`listeneradded_56f70cc2`STRING,`ddginputtype`STRING,`name`STRING,`focus`STRING,`facilityId`STRING,`dashlanerid`STRING,`errors`STRING,`dashlanelabel`STRING,`index`STRING,`qbtmpid`STRING,`filtersused`STRING,`pageId`STRING,`listeneradded_5f3fd31e`STRING,`unitssorted`array,`event`STRING,`widgetId`STRING,`timespent`STRING,`tenantcity`STRING,`type`STRING,`discountedprice`STRING,`reservationid`STRING,`unittierid`STRING,`categoryused`STRING,`discountplanid`STRING,`unitsshown`STRING,`timestamp`STRING,`tenantpostal`STRING,`trackClick`STRING,`timeStamp`STRING,`gtmforminteractfieldid`STRING,`testid`STRING,`userLong`STRING,`imagesviewed`STRING,`hscfbound`STRING,`eventlocation`STRING,`sessionId`STRING,`immersivetranslatewalked`STRING,`sortorder`STRING,`buttontext`STRING,`rruidropdownitem`STRING,`facilityid`STRING,`userLat`STRING,`gramm`STRING,`discountapplied`STRING,`dashlaneclassification`STRING,`pageid`STRING,`softreservation`STRING,`category`STRING,`size`STRING,`status`STRING,`ishidden`STRING,`companyId`STRING,`discountattempted`STRING,`reservationurl`STRING,`listeneradded_d31b2f7b`STRING,`widgetname`STRING,`price`STRING,`tenantstate`STRING,`checked`STRING,`ctadumped`STRING,`nganimate`STRING,`loading`STRING,`unitgroupid`STRING,`listeneradded_8f80b699`STRING,`group`STRING,`listeneradded_1ac3268d`STRING,`actiontype`STRING,`listeneradded_b2157329`STRING,`gtmforminteractid`STRING,`userlong`STRING,`sessionid`STRING,`hover`STRING,`listeneradded_73d9f4ac`STRING,`widgetid`STRING,`trackclick`STRING,`id`STRING,`pageVersion`STRING,`companyid`STRING,`moveindate`STRING,`siteid`STRING,`buttonText`STRING,`invalid`STRING,`userlat`STRING,`pageversion`STRING\n)\nPARTITIONED BY (dt string)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n \nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo'\n",
    "MSCK REPAIR TABLE sfdl_prod_drivethru.websites_data_analytics_demo"
]
    • hive_path: s3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo
    • id: 1411920
    • processed: 0
    • raw_data: 
      {
    "companyid": "5ac68ae4-0ac1-418a-af7b-98bc192406f3",
    "event": "pageView",
    "facilityid": null,
    "name": "Home Page",
    "pageid": "6798ee28ab7c55afd66a4ff5",
    "pageversion": "67af545b-1362-462a-9fec-a38bc5ebd71c",
    "rruidropdownitem": "",
    "sessionid": "z_HWoRTJsHMBxBMFUztcRHNXa8sCGCij",
    "siteid": "64b56c713c159605b5ac5178",
    "timestamp": "2025-08-04T06:48:20.068Z",
    "type": "voyager/pages/types/HOME_PAGE",
    "userlat": 39.0572,
    "userlong": -94.5933,
    "widgetid": "cf63d503-c4ff-4165-90d5-ed8e333f52d6"
}
    • schema_attributes: 
      {
    "actiontype": "string",
    "buttonText": "string",
    "buttontext": "string",
    "category": "string",
    "categoryused": "string",
    "checked": "string",
    "companyId": "string",
    "companyid": "string",
    "ctadumped": "string",
    "dashlaneclassification": "string",
    "dashlanelabel": "string",
    "dashlanerid": "string",
    "ddginputtype": "string",
    "discountapplied": "string",
    "discountattempted": "string",
    "discountedprice": "string",
    "discountplanid": "string",
    "errors": "string",
    "event": "string",
    "eventlocation": "string",
    "facilityId": "string",
    "facilityid": "string",
    "filtersused": "string",
    "focus": "string",
    "gramm": "string",
    "group": "string",
    "gtmforminteractfieldid": "string",
    "gtmforminteractid": "string",
    "hover": "string",
    "hscfbound": "string",
    "id": "string",
    "imagesviewed": "string",
    "immersivetranslatewalked": "string",
    "index": "string",
    "invalid": "string",
    "ishidden": "string",
    "listeneradded_1ac3268d": "string",
    "listeneradded_56f70cc2": "string",
    "listeneradded_5f3fd31e": "string",
    "listeneradded_73d9f4ac": "string",
    "listeneradded_8f80b699": "string",
    "listeneradded_b2157329": "string",
    "listeneradded_d31b2f7b": "string",
    "loading": "string",
    "moveindate": "string",
    "name": "string",
    "nganimate": "string",
    "pageId": "string",
    "pageVersion": "string",
    "pageid": "string",
    "pageversion": "string",
    "price": "string",
    "qbtmpid": "string",
    "reservationid": "string",
    "reservationurl": "string",
    "rruidropdownitem": "string",
    "sessionId": "string",
    "sessionid": "string",
    "siteId": "string",
    "siteid": "string",
    "size": "string",
    "softreservation": "string",
    "sortorder": "string",
    "status": "string",
    "tenantcity": "string",
    "tenantpostal": "string",
    "tenantstate": "string",
    "testid": "string",
    "timeStamp": "string",
    "timespent": "string",
    "timestamp": "string",
    "trackClick": "string",
    "trackclick": "string",
    "type": "string",
    "unitgroupid": "string",
    "unitsshown": "string",
    "unitssorted": [
        "string"
    ],
    "unittierid": "string",
    "userLat": "string",
    "userLong": "string",
    "userlat": "string",
    "userlong": "string",
    "widgetId": "string",
    "widgetid": "string",
    "widgetname": "string"
}
    • schema_name_id: 3476
    • schema_scan_id: 146906049
    • updated: None
    • created: 2025-07-10 04:05:10
    • ddl: 
      [
    "CREATE DATABASE IF NOT EXISTS `sfdl_prod_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_prod_drivethru.websites_data_analytics_demo`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_prod_drivethru.websites_data_analytics_demo`\n(\n`siteId`STRING,`listeneradded_56f70cc2`STRING,`ddginputtype`STRING,`name`STRING,`focus`STRING,`facilityId`STRING,`dashlanerid`STRING,`errors`STRING,`dashlanelabel`STRING,`index`STRING,`qbtmpid`STRING,`filtersused`STRING,`pageId`STRING,`listeneradded_5f3fd31e`STRING,`unitssorted`array,`event`STRING,`widgetId`STRING,`tenantcity`STRING,`type`STRING,`discountedprice`STRING,`reservationid`STRING,`unittierid`STRING,`categoryused`STRING,`discountplanid`STRING,`unitsshown`STRING,`timestamp`STRING,`tenantpostal`STRING,`trackClick`STRING,`timeStamp`STRING,`gtmforminteractfieldid`STRING,`testid`STRING,`userLong`STRING,`imagesviewed`STRING,`hscfbound`STRING,`eventlocation`STRING,`sessionId`STRING,`immersivetranslatewalked`STRING,`buttontext`STRING,`facilityid`STRING,`userLat`STRING,`gramm`STRING,`discountapplied`STRING,`dashlaneclassification`STRING,`pageid`STRING,`softreservation`STRING,`category`STRING,`size`STRING,`status`STRING,`ishidden`STRING,`userlat`STRING,`companyId`STRING,`discountattempted`STRING,`reservationurl`STRING,`listeneradded_d31b2f7b`STRING,`widgetname`STRING,`price`STRING,`tenantstate`STRING,`checked`STRING,`ctadumped`STRING,`nganimate`STRING,`loading`STRING,`unitgroupid`STRING,`group`STRING,`listeneradded_8f80b699`STRING,`listeneradded_1ac3268d`STRING,`actiontype`STRING,`listeneradded_b2157329`STRING,`gtmforminteractid`STRING,`userlong`STRING,`sessionid`STRING,`hover`STRING,`listeneradded_73d9f4ac`STRING,`widgetid`STRING,`trackclick`STRING,`pageversion`STRING,`id`STRING,`companyid`STRING,`pageVersion`STRING,`moveindate`STRING,`siteid`STRING,`buttonText`STRING,`invalid`STRING,`sortorder`STRING,`timespent`STRING\n)\nPARTITIONED BY (dt string)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n \nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo'\n",
    "MSCK REPAIR TABLE sfdl_prod_drivethru.websites_data_analytics_demo"
]
    • hive_path: s3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo
    • id: 1381213
    • processed: 0
    • raw_data: 
      {
    "companyid": "60f040b3-fbcd-4c2d-892f-62673b3ec7ef",
    "event": "pageView",
    "facilityid": null,
    "hover": "",
    "listeneradded_8f80b699": "true",
    "name": "Self Storage Units | NJ, NY, PA, and AZ",
    "pageid": "5bc768f99871560676d89e83",
    "pageversion": "30673a8e-12a7-4529-b8fd-b69546421444",
    "sessionid": "R05S-C_VsvrfIyKAsqHgLhGkvsbFmFdK",
    "siteid": "5bc768d1417f9dfb8aa5e0e1",
    "timestamp": "2025-07-10T08:46:17.679Z",
    "type": "voyager/pages/types/HOME_PAGE",
    "userlat": 39.0572,
    "userlong": -94.5933,
    "widgetid": "bdad6ae1-97e6-4b9a-b18c-d251ca6e2356"
}
    • schema_attributes: 
      {
    "actiontype": "string",
    "buttonText": "string",
    "buttontext": "string",
    "category": "string",
    "categoryused": "string",
    "checked": "string",
    "companyId": "string",
    "companyid": "string",
    "ctadumped": "string",
    "dashlaneclassification": "string",
    "dashlanelabel": "string",
    "dashlanerid": "string",
    "ddginputtype": "string",
    "discountapplied": "string",
    "discountattempted": "string",
    "discountedprice": "string",
    "discountplanid": "string",
    "errors": "string",
    "event": "string",
    "eventlocation": "string",
    "facilityId": "string",
    "facilityid": "string",
    "filtersused": "string",
    "focus": "string",
    "gramm": "string",
    "group": "string",
    "gtmforminteractfieldid": "string",
    "gtmforminteractid": "string",
    "hover": "string",
    "hscfbound": "string",
    "id": "string",
    "imagesviewed": "string",
    "immersivetranslatewalked": "string",
    "index": "string",
    "invalid": "string",
    "ishidden": "string",
    "listeneradded_1ac3268d": "string",
    "listeneradded_56f70cc2": "string",
    "listeneradded_5f3fd31e": "string",
    "listeneradded_73d9f4ac": "string",
    "listeneradded_8f80b699": "string",
    "listeneradded_b2157329": "string",
    "listeneradded_d31b2f7b": "string",
    "loading": "string",
    "moveindate": "string",
    "name": "string",
    "nganimate": "string",
    "pageId": "string",
    "pageVersion": "string",
    "pageid": "string",
    "pageversion": "string",
    "price": "string",
    "qbtmpid": "string",
    "reservationid": "string",
    "reservationurl": "string",
    "sessionId": "string",
    "sessionid": "string",
    "siteId": "string",
    "siteid": "string",
    "size": "string",
    "softreservation": "string",
    "sortorder": "string",
    "status": "string",
    "tenantcity": "string",
    "tenantpostal": "string",
    "tenantstate": "string",
    "testid": "string",
    "timeStamp": "string",
    "timespent": "string",
    "timestamp": "string",
    "trackClick": "string",
    "trackclick": "string",
    "type": "string",
    "unitgroupid": "string",
    "unitsshown": "string",
    "unitssorted": [
        "string"
    ],
    "unittierid": "string",
    "userLat": "string",
    "userLong": "string",
    "userlat": "string",
    "userlong": "string",
    "widgetId": "string",
    "widgetid": "string",
    "widgetname": "string"
}
    • schema_name_id: 3476
    • schema_scan_id: 145361467
    • updated: None
    • created: 2025-07-03 05:05:17
    • ddl: 
      [
    "CREATE DATABASE IF NOT EXISTS `sfdl_prod_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_prod_drivethru.websites_data_analytics_demo`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_prod_drivethru.websites_data_analytics_demo`\n(\n`siteId`STRING,`listeneradded_56f70cc2`STRING,`ddginputtype`STRING,`name`STRING,`focus`STRING,`facilityId`STRING,`dashlanerid`STRING,`errors`STRING,`dashlanelabel`STRING,`index`STRING,`qbtmpid`STRING,`filtersused`STRING,`pageId`STRING,`listeneradded_5f3fd31e`STRING,`unitssorted`array,`event`STRING,`widgetId`STRING,`timespent`STRING,`tenantcity`STRING,`type`STRING,`discountedprice`STRING,`reservationid`STRING,`unittierid`STRING,`categoryused`STRING,`discountplanid`STRING,`unitsshown`STRING,`timestamp`STRING,`tenantpostal`STRING,`trackClick`STRING,`timeStamp`STRING,`gtmforminteractfieldid`STRING,`testid`STRING,`userLong`STRING,`imagesviewed`STRING,`hscfbound`STRING,`eventlocation`STRING,`sessionId`STRING,`immersivetranslatewalked`STRING,`sortorder`STRING,`buttontext`STRING,`facilityid`STRING,`userLat`STRING,`gramm`STRING,`discountapplied`STRING,`dashlaneclassification`STRING,`pageid`STRING,`softreservation`STRING,`category`STRING,`size`STRING,`status`STRING,`ishidden`STRING,`companyId`STRING,`discountattempted`STRING,`reservationurl`STRING,`listeneradded_d31b2f7b`STRING,`widgetname`STRING,`price`STRING,`tenantstate`STRING,`checked`STRING,`ctadumped`STRING,`nganimate`STRING,`loading`STRING,`unitgroupid`STRING,`group`STRING,`listeneradded_1ac3268d`STRING,`actiontype`STRING,`gtmforminteractid`STRING,`userlong`STRING,`sessionid`STRING,`widgetid`STRING,`hover`STRING,`listeneradded_73d9f4ac`STRING,`listeneradded_b2157329`STRING,`trackclick`STRING,`id`STRING,`pageVersion`STRING,`companyid`STRING,`moveindate`STRING,`siteid`STRING,`buttonText`STRING,`invalid`STRING,`userlat`STRING,`pageversion`STRING\n)\nPARTITIONED BY (dt string)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n \nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo'\n",
    "MSCK REPAIR TABLE sfdl_prod_drivethru.websites_data_analytics_demo"
]
    • hive_path: s3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo
    • id: 1372087
    • processed: 0
    • raw_data: 
      {
    "companyid": "9216f58a-0165-4889-816c-f796a3dcfee8",
    "event": "pageView",
    "facilityid": null,
    "hover": "",
    "listeneradded_56f70cc2": "true",
    "listeneradded_b2157329": "true",
    "name": "Storage Units in Arkansas",
    "pageid": "5af1bfcaf5023e05fb8ad355",
    "pageversion": "7d4fc9f5-602d-4bbe-853c-3f24acf13915",
    "sessionid": "kx-u6fhhgpdTb0H9DMhbQOTLpgi01S-S",
    "siteid": "5af1bfbc9ed312df2d066d54",
    "timestamp": "2025-07-03T09:45:23.158Z",
    "type": "voyager/pages/types/HOME_PAGE",
    "userlat": 39.0572,
    "userlong": -94.5933,
    "widgetid": "bdad6ae1-97e6-4b9a-b18c-d251ca6e2356"
}
    • schema_attributes: 
      {
    "actiontype": "string",
    "buttonText": "string",
    "buttontext": "string",
    "category": "string",
    "categoryused": "string",
    "checked": "string",
    "companyId": "string",
    "companyid": "string",
    "ctadumped": "string",
    "dashlaneclassification": "string",
    "dashlanelabel": "string",
    "dashlanerid": "string",
    "ddginputtype": "string",
    "discountapplied": "string",
    "discountattempted": "string",
    "discountedprice": "string",
    "discountplanid": "string",
    "errors": "string",
    "event": "string",
    "eventlocation": "string",
    "facilityId": "string",
    "facilityid": "string",
    "filtersused": "string",
    "focus": "string",
    "gramm": "string",
    "group": "string",
    "gtmforminteractfieldid": "string",
    "gtmforminteractid": "string",
    "hover": "string",
    "hscfbound": "string",
    "id": "string",
    "imagesviewed": "string",
    "immersivetranslatewalked": "string",
    "index": "string",
    "invalid": "string",
    "ishidden": "string",
    "listeneradded_1ac3268d": "string",
    "listeneradded_56f70cc2": "string",
    "listeneradded_5f3fd31e": "string",
    "listeneradded_73d9f4ac": "string",
    "listeneradded_b2157329": "string",
    "listeneradded_d31b2f7b": "string",
    "loading": "string",
    "moveindate": "string",
    "name": "string",
    "nganimate": "string",
    "pageId": "string",
    "pageVersion": "string",
    "pageid": "string",
    "pageversion": "string",
    "price": "string",
    "qbtmpid": "string",
    "reservationid": "string",
    "reservationurl": "string",
    "sessionId": "string",
    "sessionid": "string",
    "siteId": "string",
    "siteid": "string",
    "size": "string",
    "softreservation": "string",
    "sortorder": "string",
    "status": "string",
    "tenantcity": "string",
    "tenantpostal": "string",
    "tenantstate": "string",
    "testid": "string",
    "timeStamp": "string",
    "timespent": "string",
    "timestamp": "string",
    "trackClick": "string",
    "trackclick": "string",
    "type": "string",
    "unitgroupid": "string",
    "unitsshown": "string",
    "unitssorted": [
        "string"
    ],
    "unittierid": "string",
    "userLat": "string",
    "userLong": "string",
    "userlat": "string",
    "userlong": "string",
    "widgetId": "string",
    "widgetid": "string",
    "widgetname": "string"
}
    • schema_name_id: 3476
    • schema_scan_id: 144955230
    • updated: None
Schema Scans: 3
Last at 2025-08-04 02:05:12
    • duration: 0:00:12.230970
    • exit_message: 
      {
    "exception": "failed to apply schema `Query failed: FAILED: SemanticException [Error 10036]: Duplicate column name: timestamp `CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_prod_drivethru.websites_data_analytics_demo`\n(\n`siteId`STRING,`listeneradded_56f70cc2`STRING,`ddginputtype`STRING,`name`STRING,`focus`STRING,`facilityId`STRING,`dashlanerid`STRING,`errors`STRING,`dashlanelabel`STRING,`index`STRING,`qbtmpid`STRING,`filtersused`STRING,`pageId`STRING,`listeneradded_5f3fd31e`STRING,`unitssorted`array,`event`STRING,`widgetId`STRING,`timespent`STRING,`tenantcity`STRING,`type`STRING,`discountedprice`STRING,`reservationid`STRING,`unittierid`STRING,`categoryused`STRING,`discountplanid`STRING,`unitsshown`STRING,`timestamp`STRING,`tenantpostal`STRING,`trackClick`STRING,`timeStamp`STRING,`gtmforminteractfieldid`STRING,`testid`STRING,`userLong`STRING,`imagesviewed`STRING,`hscfbound`STRING,`eventlocation`STRING,`sessionId`STRING,`immersivetranslatewalked`STRING,`sortorder`STRING,`buttontext`STRING,`rruidropdownitem`STRING,`facilityid`STRING,`userLat`STRING,`gramm`STRING,`discountapplied`STRING,`dashlaneclassification`STRING,`pageid`STRING,`softreservation`STRING,`category`STRING,`size`STRING,`status`STRING,`ishidden`STRING,`companyId`STRING,`discountattempted`STRING,`reservationurl`STRING,`listeneradded_d31b2f7b`STRING,`widgetname`STRING,`price`STRING,`tenantstate`STRING,`checked`STRING,`ctadumped`STRING,`nganimate`STRING,`loading`STRING,`unitgroupid`STRING,`listeneradded_8f80b699`STRING,`group`STRING,`listeneradded_1ac3268d`STRING,`actiontype`STRING,`listeneradded_b2157329`STRING,`gtmforminteractid`STRING,`userlong`STRING,`sessionid`STRING,`hover`STRING,`listeneradded_73d9f4ac`STRING,`widgetid`STRING,`trackclick`STRING,`id`STRING,`pageVersion`STRING,`companyid`STRING,`moveindate`STRING,`siteid`STRING,`buttonText`STRING,`invalid`STRING,`userlat`STRING,`pageversion`STRING\n)\nPARTITIONED BY (dt string)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n \nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo'\n``: ['CREATE DATABASE IF NOT EXISTS `sfdl_prod_drivethru`', 'DROP TABLE IF EXISTS `sfdl_prod_drivethru.websites_data_analytics_demo`', \"CREATE EXTERNAL TABLE IF NOT EXISTS\\n`sfdl_prod_drivethru.websites_data_analytics_demo`\\n(\\n`siteId`STRING,`listeneradded_56f70cc2`STRING,`ddginputtype`STRING,`name`STRING,`focus`STRING,`facilityId`STRING,`dashlanerid`STRING,`errors`STRING,`dashlanelabel`STRING,`index`STRING,`qbtmpid`STRING,`filtersused`STRING,`pageId`STRING,`listeneradded_5f3fd31e`STRING,`unitssorted`array,`event`STRING,`widgetId`STRING,`timespent`STRING,`tenantcity`STRING,`type`STRING,`discountedprice`STRING,`reservationid`STRING,`unittierid`STRING,`categoryused`STRING,`discountplanid`STRING,`unitsshown`STRING,`timestamp`STRING,`tenantpostal`STRING,`trackClick`STRING,`timeStamp`STRING,`gtmforminteractfieldid`STRING,`testid`STRING,`userLong`STRING,`imagesviewed`STRING,`hscfbound`STRING,`eventlocation`STRING,`sessionId`STRING,`immersivetranslatewalked`STRING,`sortorder`STRING,`buttontext`STRING,`rruidropdownitem`STRING,`facilityid`STRING,`userLat`STRING,`gramm`STRING,`discountapplied`STRING,`dashlaneclassification`STRING,`pageid`STRING,`softreservation`STRING,`category`STRING,`size`STRING,`status`STRING,`ishidden`STRING,`companyId`STRING,`discountattempted`STRING,`reservationurl`STRING,`listeneradded_d31b2f7b`STRING,`widgetname`STRING,`price`STRING,`tenantstate`STRING,`checked`STRING,`ctadumped`STRING,`nganimate`STRING,`loading`STRING,`unitgroupid`STRING,`listeneradded_8f80b699`STRING,`group`STRING,`listeneradded_1ac3268d`STRING,`actiontype`STRING,`listeneradded_b2157329`STRING,`gtmforminteractid`STRING,`userlong`STRING,`sessionid`STRING,`hover`STRING,`listeneradded_73d9f4ac`STRING,`widgetid`STRING,`trackclick`STRING,`id`STRING,`pageVersion`STRING,`companyid`STRING,`moveindate`STRING,`siteid`STRING,`buttonText`STRING,`invalid`STRING,`userlat`STRING,`pageversion`STRING\\n)\\nPARTITIONED BY (dt string)\\nROW FORMAT SERDE\\n'org.openx.data.jsonserde.JsonSerDe'\\n \\nSTORED AS INPUTFORMAT\\n  'org.apache.hadoop.mapred.TextInputFormat'\\nOUTPUTFORMAT\\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\\nLOCATION\\n  's3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo'\\n\", 'MSCK REPAIR TABLE sfdl_prod_drivethru.websites_data_analytics_demo']\nTraceback (most recent call last):\n  File \"/schema_manager/schema_generator/table_generator.py\", line 24, in apply_latest_version\n    self.runner.run_multi_query(schema_version.ddl, timeout=DDLQueueWorker.TIMEOUT)\n  File \"/usr/local/lib/python3.12/site-packages/stodl/athena_runner/runner.py\", line 133, in run_multi_query\n    result = self.run_single_query(query, timeout, database)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/local/lib/python3.12/site-packages/stodl/athena_runner/runner.py\", line 51, in run_single_query\n    return self._run_single_query(query, timeout, database)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/local/lib/python3.12/site-packages/stodl/athena_runner/runner.py\", line 120, in _run_single_query\n    raise e\n  File \"/usr/local/lib/python3.12/site-packages/stodl/athena_runner/runner.py\", line 103, in _run_single_query\n    raise AthenaRunnerQueryException('Query failed: {} `{}`'.format(info['QueryExecution']['Status']['StateChangeReason'], query))\nstodl.athena_runner.exceptions.AthenaRunnerQueryException: Query failed: FAILED: SemanticException [Error 10036]: Duplicate column name: timestamp `CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_prod_drivethru.websites_data_analytics_demo`\n(\n`siteId`STRING,`listeneradded_56f70cc2`STRING,`ddginputtype`STRING,`name`STRING,`focus`STRING,`facilityId`STRING,`dashlanerid`STRING,`errors`STRING,`dashlanelabel`STRING,`index`STRING,`qbtmpid`STRING,`filtersused`STRING,`pageId`STRING,`listeneradded_5f3fd31e`STRING,`unitssorted`array,`event`STRING,`widgetId`STRING,`timespent`STRING,`tenantcity`STRING,`type`STRING,`discountedprice`STRING,`reservationid`STRING,`unittierid`STRING,`categoryused`STRING,`discountplanid`STRING,`unitsshown`STRING,`timestamp`STRING,`tenantpostal`STRING,`trackClick`STRING,`timeStamp`STRING,`gtmforminteractfieldid`STRING,`testid`STRING,`userLong`STRING,`imagesviewed`STRING,`hscfbound`STRING,`eventlocation`STRING,`sessionId`STRING,`immersivetranslatewalked`STRING,`sortorder`STRING,`buttontext`STRING,`rruidropdownitem`STRING,`facilityid`STRING,`userLat`STRING,`gramm`STRING,`discountapplied`STRING,`dashlaneclassification`STRING,`pageid`STRING,`softreservation`STRING,`category`STRING,`size`STRING,`status`STRING,`ishidden`STRING,`companyId`STRING,`discountattempted`STRING,`reservationurl`STRING,`listeneradded_d31b2f7b`STRING,`widgetname`STRING,`price`STRING,`tenantstate`STRING,`checked`STRING,`ctadumped`STRING,`nganimate`STRING,`loading`STRING,`unitgroupid`STRING,`listeneradded_8f80b699`STRING,`group`STRING,`listeneradded_1ac3268d`STRING,`actiontype`STRING,`listeneradded_b2157329`STRING,`gtmforminteractid`STRING,`userlong`STRING,`sessionid`STRING,`hover`STRING,`listeneradded_73d9f4ac`STRING,`widgetid`STRING,`trackclick`STRING,`id`STRING,`pageVersion`STRING,`companyid`STRING,`moveindate`STRING,`siteid`STRING,`buttonText`STRING,`invalid`STRING,`userlat`STRING,`pageversion`STRING\n)\nPARTITIONED BY (dt string)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n \nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo'\n`\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File \"/schema_manager/schema_generator/schema_scanner.py\", line 102, in scan\n    response = service.run()\n               ^^^^^^^^^^^^^\n  File \"/schema_manager/schema_generator/process_hive_table.py\", line 98, in run\n    response['ddl_changed'] = table_generator.run()\n                              ^^^^^^^^^^^^^^^^^^^^^\n  File \"/schema_manager/schema_generator/table_generator.py\", line 35, in run\n    self.apply_latest_version()\n  File \"/schema_manager/schema_generator/table_generator.py\", line 26, in apply_latest_version\n    raise Exception('failed to apply schema `{}`: {}'.format(e, schema_version.ddl))\nException: failed to apply schema `Query failed: FAILED: SemanticException [Error 10036]: Duplicate column name: timestamp `CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_prod_drivethru.websites_data_analytics_demo`\n(\n`siteId`STRING,`listeneradded_56f70cc2`STRING,`ddginputtype`STRING,`name`STRING,`focus`STRING,`facilityId`STRING,`dashlanerid`STRING,`errors`STRING,`dashlanelabel`STRING,`index`STRING,`qbtmpid`STRING,`filtersused`STRING,`pageId`STRING,`listeneradded_5f3fd31e`STRING,`unitssorted`array,`event`STRING,`widgetId`STRING,`timespent`STRING,`tenantcity`STRING,`type`STRING,`discountedprice`STRING,`reservationid`STRING,`unittierid`STRING,`categoryused`STRING,`discountplanid`STRING,`unitsshown`STRING,`timestamp`STRING,`tenantpostal`STRING,`trackClick`STRING,`timeStamp`STRING,`gtmforminteractfieldid`STRING,`testid`STRING,`userLong`STRING,`imagesviewed`STRING,`hscfbound`STRING,`eventlocation`STRING,`sessionId`STRING,`immersivetranslatewalked`STRING,`sortorder`STRING,`buttontext`STRING,`rruidropdownitem`STRING,`facilityid`STRING,`userLat`STRING,`gramm`STRING,`discountapplied`STRING,`dashlaneclassification`STRING,`pageid`STRING,`softreservation`STRING,`category`STRING,`size`STRING,`status`STRING,`ishidden`STRING,`companyId`STRING,`discountattempted`STRING,`reservationurl`STRING,`listeneradded_d31b2f7b`STRING,`widgetname`STRING,`price`STRING,`tenantstate`STRING,`checked`STRING,`ctadumped`STRING,`nganimate`STRING,`loading`STRING,`unitgroupid`STRING,`listeneradded_8f80b699`STRING,`group`STRING,`listeneradded_1ac3268d`STRING,`actiontype`STRING,`listeneradded_b2157329`STRING,`gtmforminteractid`STRING,`userlong`STRING,`sessionid`STRING,`hover`STRING,`listeneradded_73d9f4ac`STRING,`widgetid`STRING,`trackclick`STRING,`id`STRING,`pageVersion`STRING,`companyid`STRING,`moveindate`STRING,`siteid`STRING,`buttonText`STRING,`invalid`STRING,`userlat`STRING,`pageversion`STRING\n)\nPARTITIONED BY (dt string)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n \nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo'\n``: ['CREATE DATABASE IF NOT EXISTS `sfdl_prod_drivethru`', 'DROP TABLE IF EXISTS `sfdl_prod_drivethru.websites_data_analytics_demo`', \"CREATE EXTERNAL TABLE IF NOT EXISTS\\n`sfdl_prod_drivethru.websites_data_analytics_demo`\\n(\\n`siteId`STRING,`listeneradded_56f70cc2`STRING,`ddginputtype`STRING,`name`STRING,`focus`STRING,`facilityId`STRING,`dashlanerid`STRING,`errors`STRING,`dashlanelabel`STRING,`index`STRING,`qbtmpid`STRING,`filtersused`STRING,`pageId`STRING,`listeneradded_5f3fd31e`STRING,`unitssorted`array,`event`STRING,`widgetId`STRING,`timespent`STRING,`tenantcity`STRING,`type`STRING,`discountedprice`STRING,`reservationid`STRING,`unittierid`STRING,`categoryused`STRING,`discountplanid`STRING,`unitsshown`STRING,`timestamp`STRING,`tenantpostal`STRING,`trackClick`STRING,`timeStamp`STRING,`gtmforminteractfieldid`STRING,`testid`STRING,`userLong`STRING,`imagesviewed`STRING,`hscfbound`STRING,`eventlocation`STRING,`sessionId`STRING,`immersivetranslatewalked`STRING,`sortorder`STRING,`buttontext`STRING,`rruidropdownitem`STRING,`facilityid`STRING,`userLat`STRING,`gramm`STRING,`discountapplied`STRING,`dashlaneclassification`STRING,`pageid`STRING,`softreservation`STRING,`category`STRING,`size`STRING,`status`STRING,`ishidden`STRING,`companyId`STRING,`discountattempted`STRING,`reservationurl`STRING,`listeneradded_d31b2f7b`STRING,`widgetname`STRING,`price`STRING,`tenantstate`STRING,`checked`STRING,`ctadumped`STRING,`nganimate`STRING,`loading`STRING,`unitgroupid`STRING,`listeneradded_8f80b699`STRING,`group`STRING,`listeneradded_1ac3268d`STRING,`actiontype`STRING,`listeneradded_b2157329`STRING,`gtmforminteractid`STRING,`userlong`STRING,`sessionid`STRING,`hover`STRING,`listeneradded_73d9f4ac`STRING,`widgetid`STRING,`trackclick`STRING,`id`STRING,`pageVersion`STRING,`companyid`STRING,`moveindate`STRING,`siteid`STRING,`buttonText`STRING,`invalid`STRING,`userlat`STRING,`pageversion`STRING\\n)\\nPARTITIONED BY (dt string)\\nROW FORMAT SERDE\\n'org.openx.data.jsonserde.JsonSerDe'\\n \\nSTORED AS INPUTFORMAT\\n  'org.apache.hadoop.mapred.TextInputFormat'\\nOUTPUTFORMAT\\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\\nLOCATION\\n  's3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo'\\n\", 'MSCK REPAIR TABLE sfdl_prod_drivethru.websites_data_analytics_demo']\n: Expecting value: line 1 column 1 (char 0)"
}
    • id: 146906049
    • payload: 
      {
    "datatype_dict": null,
    "file_format": "json",
    "is_schema_datatyped": true,
    "partition": {
        "location": "s3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo/dt=2025-08-04",
        "name": "dt",
        "type": "string",
        "value": "2025-08-04"
    },
    "s3": {
        "bucket": "sfdl-drivethrujson-drivethru-prod",
        "hive_path": "s3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo",
        "key": "raw_incremental/websites_data_analytics_demo/dt=2025-08-04/events_02_00.json.gz"
    },
    "schema": {
        "name": "sfdl_prod_drivethru.websites_data_analytics_demo",
        "type": "drivethrujson"
    },
    "trace_context": {
        "parent_id": 13118314481783867873,
        "service_name": "drive-thru",
        "span_id": 5857471475695265028,
        "trace_id": 14833480915907496149
    }
}
    • running: False
    • schema_name_id: 3476
    • start_time: 2025-08-04 02:05:12
    • success: False
    • trace_id: 14833480915907496149
    • duration: 0:00:12.299462
    • exit_message: 
      {
    "exception": "failed to apply schema `Query failed: FAILED: SemanticException [Error 10036]: Duplicate column name: timestamp `CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_prod_drivethru.websites_data_analytics_demo`\n(\n`siteId`STRING,`listeneradded_56f70cc2`STRING,`ddginputtype`STRING,`name`STRING,`focus`STRING,`facilityId`STRING,`dashlanerid`STRING,`errors`STRING,`dashlanelabel`STRING,`index`STRING,`qbtmpid`STRING,`filtersused`STRING,`pageId`STRING,`listeneradded_5f3fd31e`STRING,`unitssorted`array,`event`STRING,`widgetId`STRING,`tenantcity`STRING,`type`STRING,`discountedprice`STRING,`reservationid`STRING,`unittierid`STRING,`categoryused`STRING,`discountplanid`STRING,`unitsshown`STRING,`timestamp`STRING,`tenantpostal`STRING,`trackClick`STRING,`timeStamp`STRING,`gtmforminteractfieldid`STRING,`testid`STRING,`userLong`STRING,`imagesviewed`STRING,`hscfbound`STRING,`eventlocation`STRING,`sessionId`STRING,`immersivetranslatewalked`STRING,`buttontext`STRING,`facilityid`STRING,`userLat`STRING,`gramm`STRING,`discountapplied`STRING,`dashlaneclassification`STRING,`pageid`STRING,`softreservation`STRING,`category`STRING,`size`STRING,`status`STRING,`ishidden`STRING,`userlat`STRING,`companyId`STRING,`discountattempted`STRING,`reservationurl`STRING,`listeneradded_d31b2f7b`STRING,`widgetname`STRING,`price`STRING,`tenantstate`STRING,`checked`STRING,`ctadumped`STRING,`nganimate`STRING,`loading`STRING,`unitgroupid`STRING,`group`STRING,`listeneradded_8f80b699`STRING,`listeneradded_1ac3268d`STRING,`actiontype`STRING,`listeneradded_b2157329`STRING,`gtmforminteractid`STRING,`userlong`STRING,`sessionid`STRING,`hover`STRING,`listeneradded_73d9f4ac`STRING,`widgetid`STRING,`trackclick`STRING,`pageversion`STRING,`id`STRING,`companyid`STRING,`pageVersion`STRING,`moveindate`STRING,`siteid`STRING,`buttonText`STRING,`invalid`STRING,`sortorder`STRING,`timespent`STRING\n)\nPARTITIONED BY (dt string)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n \nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo'\n``: ['CREATE DATABASE IF NOT EXISTS `sfdl_prod_drivethru`', 'DROP TABLE IF EXISTS `sfdl_prod_drivethru.websites_data_analytics_demo`', \"CREATE EXTERNAL TABLE IF NOT EXISTS\\n`sfdl_prod_drivethru.websites_data_analytics_demo`\\n(\\n`siteId`STRING,`listeneradded_56f70cc2`STRING,`ddginputtype`STRING,`name`STRING,`focus`STRING,`facilityId`STRING,`dashlanerid`STRING,`errors`STRING,`dashlanelabel`STRING,`index`STRING,`qbtmpid`STRING,`filtersused`STRING,`pageId`STRING,`listeneradded_5f3fd31e`STRING,`unitssorted`array,`event`STRING,`widgetId`STRING,`tenantcity`STRING,`type`STRING,`discountedprice`STRING,`reservationid`STRING,`unittierid`STRING,`categoryused`STRING,`discountplanid`STRING,`unitsshown`STRING,`timestamp`STRING,`tenantpostal`STRING,`trackClick`STRING,`timeStamp`STRING,`gtmforminteractfieldid`STRING,`testid`STRING,`userLong`STRING,`imagesviewed`STRING,`hscfbound`STRING,`eventlocation`STRING,`sessionId`STRING,`immersivetranslatewalked`STRING,`buttontext`STRING,`facilityid`STRING,`userLat`STRING,`gramm`STRING,`discountapplied`STRING,`dashlaneclassification`STRING,`pageid`STRING,`softreservation`STRING,`category`STRING,`size`STRING,`status`STRING,`ishidden`STRING,`userlat`STRING,`companyId`STRING,`discountattempted`STRING,`reservationurl`STRING,`listeneradded_d31b2f7b`STRING,`widgetname`STRING,`price`STRING,`tenantstate`STRING,`checked`STRING,`ctadumped`STRING,`nganimate`STRING,`loading`STRING,`unitgroupid`STRING,`group`STRING,`listeneradded_8f80b699`STRING,`listeneradded_1ac3268d`STRING,`actiontype`STRING,`listeneradded_b2157329`STRING,`gtmforminteractid`STRING,`userlong`STRING,`sessionid`STRING,`hover`STRING,`listeneradded_73d9f4ac`STRING,`widgetid`STRING,`trackclick`STRING,`pageversion`STRING,`id`STRING,`companyid`STRING,`pageVersion`STRING,`moveindate`STRING,`siteid`STRING,`buttonText`STRING,`invalid`STRING,`sortorder`STRING,`timespent`STRING\\n)\\nPARTITIONED BY (dt string)\\nROW FORMAT SERDE\\n'org.openx.data.jsonserde.JsonSerDe'\\n \\nSTORED AS INPUTFORMAT\\n  'org.apache.hadoop.mapred.TextInputFormat'\\nOUTPUTFORMAT\\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\\nLOCATION\\n  's3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo'\\n\", 'MSCK REPAIR TABLE sfdl_prod_drivethru.websites_data_analytics_demo']\nTraceback (most recent call last):\n  File \"/schema_manager/schema_generator/table_generator.py\", line 24, in apply_latest_version\n    self.runner.run_multi_query(schema_version.ddl, timeout=DDLQueueWorker.TIMEOUT)\n  File \"/usr/local/lib/python3.12/site-packages/stodl/athena_runner/runner.py\", line 133, in run_multi_query\n    result = self.run_single_query(query, timeout, database)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/local/lib/python3.12/site-packages/stodl/athena_runner/runner.py\", line 51, in run_single_query\n    return self._run_single_query(query, timeout, database)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/local/lib/python3.12/site-packages/stodl/athena_runner/runner.py\", line 120, in _run_single_query\n    raise e\n  File \"/usr/local/lib/python3.12/site-packages/stodl/athena_runner/runner.py\", line 103, in _run_single_query\n    raise AthenaRunnerQueryException('Query failed: {} `{}`'.format(info['QueryExecution']['Status']['StateChangeReason'], query))\nstodl.athena_runner.exceptions.AthenaRunnerQueryException: Query failed: FAILED: SemanticException [Error 10036]: Duplicate column name: timestamp `CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_prod_drivethru.websites_data_analytics_demo`\n(\n`siteId`STRING,`listeneradded_56f70cc2`STRING,`ddginputtype`STRING,`name`STRING,`focus`STRING,`facilityId`STRING,`dashlanerid`STRING,`errors`STRING,`dashlanelabel`STRING,`index`STRING,`qbtmpid`STRING,`filtersused`STRING,`pageId`STRING,`listeneradded_5f3fd31e`STRING,`unitssorted`array,`event`STRING,`widgetId`STRING,`tenantcity`STRING,`type`STRING,`discountedprice`STRING,`reservationid`STRING,`unittierid`STRING,`categoryused`STRING,`discountplanid`STRING,`unitsshown`STRING,`timestamp`STRING,`tenantpostal`STRING,`trackClick`STRING,`timeStamp`STRING,`gtmforminteractfieldid`STRING,`testid`STRING,`userLong`STRING,`imagesviewed`STRING,`hscfbound`STRING,`eventlocation`STRING,`sessionId`STRING,`immersivetranslatewalked`STRING,`buttontext`STRING,`facilityid`STRING,`userLat`STRING,`gramm`STRING,`discountapplied`STRING,`dashlaneclassification`STRING,`pageid`STRING,`softreservation`STRING,`category`STRING,`size`STRING,`status`STRING,`ishidden`STRING,`userlat`STRING,`companyId`STRING,`discountattempted`STRING,`reservationurl`STRING,`listeneradded_d31b2f7b`STRING,`widgetname`STRING,`price`STRING,`tenantstate`STRING,`checked`STRING,`ctadumped`STRING,`nganimate`STRING,`loading`STRING,`unitgroupid`STRING,`group`STRING,`listeneradded_8f80b699`STRING,`listeneradded_1ac3268d`STRING,`actiontype`STRING,`listeneradded_b2157329`STRING,`gtmforminteractid`STRING,`userlong`STRING,`sessionid`STRING,`hover`STRING,`listeneradded_73d9f4ac`STRING,`widgetid`STRING,`trackclick`STRING,`pageversion`STRING,`id`STRING,`companyid`STRING,`pageVersion`STRING,`moveindate`STRING,`siteid`STRING,`buttonText`STRING,`invalid`STRING,`sortorder`STRING,`timespent`STRING\n)\nPARTITIONED BY (dt string)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n \nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo'\n`\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File \"/schema_manager/schema_generator/schema_scanner.py\", line 102, in scan\n    response = service.run()\n               ^^^^^^^^^^^^^\n  File \"/schema_manager/schema_generator/process_hive_table.py\", line 98, in run\n    response['ddl_changed'] = table_generator.run()\n                              ^^^^^^^^^^^^^^^^^^^^^\n  File \"/schema_manager/schema_generator/table_generator.py\", line 35, in run\n    self.apply_latest_version()\n  File \"/schema_manager/schema_generator/table_generator.py\", line 26, in apply_latest_version\n    raise Exception('failed to apply schema `{}`: {}'.format(e, schema_version.ddl))\nException: failed to apply schema `Query failed: FAILED: SemanticException [Error 10036]: Duplicate column name: timestamp `CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_prod_drivethru.websites_data_analytics_demo`\n(\n`siteId`STRING,`listeneradded_56f70cc2`STRING,`ddginputtype`STRING,`name`STRING,`focus`STRING,`facilityId`STRING,`dashlanerid`STRING,`errors`STRING,`dashlanelabel`STRING,`index`STRING,`qbtmpid`STRING,`filtersused`STRING,`pageId`STRING,`listeneradded_5f3fd31e`STRING,`unitssorted`array,`event`STRING,`widgetId`STRING,`tenantcity`STRING,`type`STRING,`discountedprice`STRING,`reservationid`STRING,`unittierid`STRING,`categoryused`STRING,`discountplanid`STRING,`unitsshown`STRING,`timestamp`STRING,`tenantpostal`STRING,`trackClick`STRING,`timeStamp`STRING,`gtmforminteractfieldid`STRING,`testid`STRING,`userLong`STRING,`imagesviewed`STRING,`hscfbound`STRING,`eventlocation`STRING,`sessionId`STRING,`immersivetranslatewalked`STRING,`buttontext`STRING,`facilityid`STRING,`userLat`STRING,`gramm`STRING,`discountapplied`STRING,`dashlaneclassification`STRING,`pageid`STRING,`softreservation`STRING,`category`STRING,`size`STRING,`status`STRING,`ishidden`STRING,`userlat`STRING,`companyId`STRING,`discountattempted`STRING,`reservationurl`STRING,`listeneradded_d31b2f7b`STRING,`widgetname`STRING,`price`STRING,`tenantstate`STRING,`checked`STRING,`ctadumped`STRING,`nganimate`STRING,`loading`STRING,`unitgroupid`STRING,`group`STRING,`listeneradded_8f80b699`STRING,`listeneradded_1ac3268d`STRING,`actiontype`STRING,`listeneradded_b2157329`STRING,`gtmforminteractid`STRING,`userlong`STRING,`sessionid`STRING,`hover`STRING,`listeneradded_73d9f4ac`STRING,`widgetid`STRING,`trackclick`STRING,`pageversion`STRING,`id`STRING,`companyid`STRING,`pageVersion`STRING,`moveindate`STRING,`siteid`STRING,`buttonText`STRING,`invalid`STRING,`sortorder`STRING,`timespent`STRING\n)\nPARTITIONED BY (dt string)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n \nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo'\n``: ['CREATE DATABASE IF NOT EXISTS `sfdl_prod_drivethru`', 'DROP TABLE IF EXISTS `sfdl_prod_drivethru.websites_data_analytics_demo`', \"CREATE EXTERNAL TABLE IF NOT EXISTS\\n`sfdl_prod_drivethru.websites_data_analytics_demo`\\n(\\n`siteId`STRING,`listeneradded_56f70cc2`STRING,`ddginputtype`STRING,`name`STRING,`focus`STRING,`facilityId`STRING,`dashlanerid`STRING,`errors`STRING,`dashlanelabel`STRING,`index`STRING,`qbtmpid`STRING,`filtersused`STRING,`pageId`STRING,`listeneradded_5f3fd31e`STRING,`unitssorted`array,`event`STRING,`widgetId`STRING,`tenantcity`STRING,`type`STRING,`discountedprice`STRING,`reservationid`STRING,`unittierid`STRING,`categoryused`STRING,`discountplanid`STRING,`unitsshown`STRING,`timestamp`STRING,`tenantpostal`STRING,`trackClick`STRING,`timeStamp`STRING,`gtmforminteractfieldid`STRING,`testid`STRING,`userLong`STRING,`imagesviewed`STRING,`hscfbound`STRING,`eventlocation`STRING,`sessionId`STRING,`immersivetranslatewalked`STRING,`buttontext`STRING,`facilityid`STRING,`userLat`STRING,`gramm`STRING,`discountapplied`STRING,`dashlaneclassification`STRING,`pageid`STRING,`softreservation`STRING,`category`STRING,`size`STRING,`status`STRING,`ishidden`STRING,`userlat`STRING,`companyId`STRING,`discountattempted`STRING,`reservationurl`STRING,`listeneradded_d31b2f7b`STRING,`widgetname`STRING,`price`STRING,`tenantstate`STRING,`checked`STRING,`ctadumped`STRING,`nganimate`STRING,`loading`STRING,`unitgroupid`STRING,`group`STRING,`listeneradded_8f80b699`STRING,`listeneradded_1ac3268d`STRING,`actiontype`STRING,`listeneradded_b2157329`STRING,`gtmforminteractid`STRING,`userlong`STRING,`sessionid`STRING,`hover`STRING,`listeneradded_73d9f4ac`STRING,`widgetid`STRING,`trackclick`STRING,`pageversion`STRING,`id`STRING,`companyid`STRING,`pageVersion`STRING,`moveindate`STRING,`siteid`STRING,`buttonText`STRING,`invalid`STRING,`sortorder`STRING,`timespent`STRING\\n)\\nPARTITIONED BY (dt string)\\nROW FORMAT SERDE\\n'org.openx.data.jsonserde.JsonSerDe'\\n \\nSTORED AS INPUTFORMAT\\n  'org.apache.hadoop.mapred.TextInputFormat'\\nOUTPUTFORMAT\\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\\nLOCATION\\n  's3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo'\\n\", 'MSCK REPAIR TABLE sfdl_prod_drivethru.websites_data_analytics_demo']\n: Expecting value: line 1 column 1 (char 0)"
}
    • id: 145361467
    • payload: 
      {
    "datatype_dict": null,
    "file_format": "json",
    "is_schema_datatyped": true,
    "partition": {
        "location": "s3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo/dt=2025-07-10",
        "name": "dt",
        "type": "string",
        "value": "2025-07-10"
    },
    "s3": {
        "bucket": "sfdl-drivethrujson-drivethru-prod",
        "hive_path": "s3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo",
        "key": "raw_incremental/websites_data_analytics_demo/dt=2025-07-10/events_04_00.json.gz"
    },
    "schema": {
        "name": "sfdl_prod_drivethru.websites_data_analytics_demo",
        "type": "drivethrujson"
    },
    "trace_context": {
        "parent_id": 18125587207925957725,
        "service_name": "drive-thru",
        "span_id": 17124446251716867639,
        "trace_id": 14900667528045558467
    }
}
    • running: False
    • schema_name_id: 3476
    • start_time: 2025-07-10 04:05:09
    • success: False
    • trace_id: 14900667528045558467
    • duration: 0:00:12.283320
    • exit_message: 
      {
    "exception": "failed to apply schema `Query failed: FAILED: SemanticException [Error 10036]: Duplicate column name: timestamp `CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_prod_drivethru.websites_data_analytics_demo`\n(\n`siteId`STRING,`listeneradded_56f70cc2`STRING,`ddginputtype`STRING,`name`STRING,`focus`STRING,`facilityId`STRING,`dashlanerid`STRING,`errors`STRING,`dashlanelabel`STRING,`index`STRING,`qbtmpid`STRING,`filtersused`STRING,`pageId`STRING,`listeneradded_5f3fd31e`STRING,`unitssorted`array,`event`STRING,`widgetId`STRING,`timespent`STRING,`tenantcity`STRING,`type`STRING,`discountedprice`STRING,`reservationid`STRING,`unittierid`STRING,`categoryused`STRING,`discountplanid`STRING,`unitsshown`STRING,`timestamp`STRING,`tenantpostal`STRING,`trackClick`STRING,`timeStamp`STRING,`gtmforminteractfieldid`STRING,`testid`STRING,`userLong`STRING,`imagesviewed`STRING,`hscfbound`STRING,`eventlocation`STRING,`sessionId`STRING,`immersivetranslatewalked`STRING,`sortorder`STRING,`buttontext`STRING,`facilityid`STRING,`userLat`STRING,`gramm`STRING,`discountapplied`STRING,`dashlaneclassification`STRING,`pageid`STRING,`softreservation`STRING,`category`STRING,`size`STRING,`status`STRING,`ishidden`STRING,`companyId`STRING,`discountattempted`STRING,`reservationurl`STRING,`listeneradded_d31b2f7b`STRING,`widgetname`STRING,`price`STRING,`tenantstate`STRING,`checked`STRING,`ctadumped`STRING,`nganimate`STRING,`loading`STRING,`unitgroupid`STRING,`group`STRING,`listeneradded_1ac3268d`STRING,`actiontype`STRING,`gtmforminteractid`STRING,`userlong`STRING,`sessionid`STRING,`widgetid`STRING,`hover`STRING,`listeneradded_73d9f4ac`STRING,`listeneradded_b2157329`STRING,`trackclick`STRING,`id`STRING,`pageVersion`STRING,`companyid`STRING,`moveindate`STRING,`siteid`STRING,`buttonText`STRING,`invalid`STRING,`userlat`STRING,`pageversion`STRING\n)\nPARTITIONED BY (dt string)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n \nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo'\n``: ['CREATE DATABASE IF NOT EXISTS `sfdl_prod_drivethru`', 'DROP TABLE IF EXISTS `sfdl_prod_drivethru.websites_data_analytics_demo`', \"CREATE EXTERNAL TABLE IF NOT EXISTS\\n`sfdl_prod_drivethru.websites_data_analytics_demo`\\n(\\n`siteId`STRING,`listeneradded_56f70cc2`STRING,`ddginputtype`STRING,`name`STRING,`focus`STRING,`facilityId`STRING,`dashlanerid`STRING,`errors`STRING,`dashlanelabel`STRING,`index`STRING,`qbtmpid`STRING,`filtersused`STRING,`pageId`STRING,`listeneradded_5f3fd31e`STRING,`unitssorted`array,`event`STRING,`widgetId`STRING,`timespent`STRING,`tenantcity`STRING,`type`STRING,`discountedprice`STRING,`reservationid`STRING,`unittierid`STRING,`categoryused`STRING,`discountplanid`STRING,`unitsshown`STRING,`timestamp`STRING,`tenantpostal`STRING,`trackClick`STRING,`timeStamp`STRING,`gtmforminteractfieldid`STRING,`testid`STRING,`userLong`STRING,`imagesviewed`STRING,`hscfbound`STRING,`eventlocation`STRING,`sessionId`STRING,`immersivetranslatewalked`STRING,`sortorder`STRING,`buttontext`STRING,`facilityid`STRING,`userLat`STRING,`gramm`STRING,`discountapplied`STRING,`dashlaneclassification`STRING,`pageid`STRING,`softreservation`STRING,`category`STRING,`size`STRING,`status`STRING,`ishidden`STRING,`companyId`STRING,`discountattempted`STRING,`reservationurl`STRING,`listeneradded_d31b2f7b`STRING,`widgetname`STRING,`price`STRING,`tenantstate`STRING,`checked`STRING,`ctadumped`STRING,`nganimate`STRING,`loading`STRING,`unitgroupid`STRING,`group`STRING,`listeneradded_1ac3268d`STRING,`actiontype`STRING,`gtmforminteractid`STRING,`userlong`STRING,`sessionid`STRING,`widgetid`STRING,`hover`STRING,`listeneradded_73d9f4ac`STRING,`listeneradded_b2157329`STRING,`trackclick`STRING,`id`STRING,`pageVersion`STRING,`companyid`STRING,`moveindate`STRING,`siteid`STRING,`buttonText`STRING,`invalid`STRING,`userlat`STRING,`pageversion`STRING\\n)\\nPARTITIONED BY (dt string)\\nROW FORMAT SERDE\\n'org.openx.data.jsonserde.JsonSerDe'\\n \\nSTORED AS INPUTFORMAT\\n  'org.apache.hadoop.mapred.TextInputFormat'\\nOUTPUTFORMAT\\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\\nLOCATION\\n  's3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo'\\n\", 'MSCK REPAIR TABLE sfdl_prod_drivethru.websites_data_analytics_demo']\nTraceback (most recent call last):\n  File \"/schema_manager/schema_generator/table_generator.py\", line 24, in apply_latest_version\n    self.runner.run_multi_query(schema_version.ddl, timeout=DDLQueueWorker.TIMEOUT)\n  File \"/usr/local/lib/python3.12/site-packages/stodl/athena_runner/runner.py\", line 133, in run_multi_query\n    result = self.run_single_query(query, timeout, database)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/local/lib/python3.12/site-packages/stodl/athena_runner/runner.py\", line 51, in run_single_query\n    return self._run_single_query(query, timeout, database)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/local/lib/python3.12/site-packages/stodl/athena_runner/runner.py\", line 120, in _run_single_query\n    raise e\n  File \"/usr/local/lib/python3.12/site-packages/stodl/athena_runner/runner.py\", line 103, in _run_single_query\n    raise AthenaRunnerQueryException('Query failed: {} `{}`'.format(info['QueryExecution']['Status']['StateChangeReason'], query))\nstodl.athena_runner.exceptions.AthenaRunnerQueryException: Query failed: FAILED: SemanticException [Error 10036]: Duplicate column name: timestamp `CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_prod_drivethru.websites_data_analytics_demo`\n(\n`siteId`STRING,`listeneradded_56f70cc2`STRING,`ddginputtype`STRING,`name`STRING,`focus`STRING,`facilityId`STRING,`dashlanerid`STRING,`errors`STRING,`dashlanelabel`STRING,`index`STRING,`qbtmpid`STRING,`filtersused`STRING,`pageId`STRING,`listeneradded_5f3fd31e`STRING,`unitssorted`array,`event`STRING,`widgetId`STRING,`timespent`STRING,`tenantcity`STRING,`type`STRING,`discountedprice`STRING,`reservationid`STRING,`unittierid`STRING,`categoryused`STRING,`discountplanid`STRING,`unitsshown`STRING,`timestamp`STRING,`tenantpostal`STRING,`trackClick`STRING,`timeStamp`STRING,`gtmforminteractfieldid`STRING,`testid`STRING,`userLong`STRING,`imagesviewed`STRING,`hscfbound`STRING,`eventlocation`STRING,`sessionId`STRING,`immersivetranslatewalked`STRING,`sortorder`STRING,`buttontext`STRING,`facilityid`STRING,`userLat`STRING,`gramm`STRING,`discountapplied`STRING,`dashlaneclassification`STRING,`pageid`STRING,`softreservation`STRING,`category`STRING,`size`STRING,`status`STRING,`ishidden`STRING,`companyId`STRING,`discountattempted`STRING,`reservationurl`STRING,`listeneradded_d31b2f7b`STRING,`widgetname`STRING,`price`STRING,`tenantstate`STRING,`checked`STRING,`ctadumped`STRING,`nganimate`STRING,`loading`STRING,`unitgroupid`STRING,`group`STRING,`listeneradded_1ac3268d`STRING,`actiontype`STRING,`gtmforminteractid`STRING,`userlong`STRING,`sessionid`STRING,`widgetid`STRING,`hover`STRING,`listeneradded_73d9f4ac`STRING,`listeneradded_b2157329`STRING,`trackclick`STRING,`id`STRING,`pageVersion`STRING,`companyid`STRING,`moveindate`STRING,`siteid`STRING,`buttonText`STRING,`invalid`STRING,`userlat`STRING,`pageversion`STRING\n)\nPARTITIONED BY (dt string)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n \nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo'\n`\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File \"/schema_manager/schema_generator/schema_scanner.py\", line 102, in scan\n    response = service.run()\n               ^^^^^^^^^^^^^\n  File \"/schema_manager/schema_generator/process_hive_table.py\", line 98, in run\n    response['ddl_changed'] = table_generator.run()\n                              ^^^^^^^^^^^^^^^^^^^^^\n  File \"/schema_manager/schema_generator/table_generator.py\", line 35, in run\n    self.apply_latest_version()\n  File \"/schema_manager/schema_generator/table_generator.py\", line 26, in apply_latest_version\n    raise Exception('failed to apply schema `{}`: {}'.format(e, schema_version.ddl))\nException: failed to apply schema `Query failed: FAILED: SemanticException [Error 10036]: Duplicate column name: timestamp `CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_prod_drivethru.websites_data_analytics_demo`\n(\n`siteId`STRING,`listeneradded_56f70cc2`STRING,`ddginputtype`STRING,`name`STRING,`focus`STRING,`facilityId`STRING,`dashlanerid`STRING,`errors`STRING,`dashlanelabel`STRING,`index`STRING,`qbtmpid`STRING,`filtersused`STRING,`pageId`STRING,`listeneradded_5f3fd31e`STRING,`unitssorted`array,`event`STRING,`widgetId`STRING,`timespent`STRING,`tenantcity`STRING,`type`STRING,`discountedprice`STRING,`reservationid`STRING,`unittierid`STRING,`categoryused`STRING,`discountplanid`STRING,`unitsshown`STRING,`timestamp`STRING,`tenantpostal`STRING,`trackClick`STRING,`timeStamp`STRING,`gtmforminteractfieldid`STRING,`testid`STRING,`userLong`STRING,`imagesviewed`STRING,`hscfbound`STRING,`eventlocation`STRING,`sessionId`STRING,`immersivetranslatewalked`STRING,`sortorder`STRING,`buttontext`STRING,`facilityid`STRING,`userLat`STRING,`gramm`STRING,`discountapplied`STRING,`dashlaneclassification`STRING,`pageid`STRING,`softreservation`STRING,`category`STRING,`size`STRING,`status`STRING,`ishidden`STRING,`companyId`STRING,`discountattempted`STRING,`reservationurl`STRING,`listeneradded_d31b2f7b`STRING,`widgetname`STRING,`price`STRING,`tenantstate`STRING,`checked`STRING,`ctadumped`STRING,`nganimate`STRING,`loading`STRING,`unitgroupid`STRING,`group`STRING,`listeneradded_1ac3268d`STRING,`actiontype`STRING,`gtmforminteractid`STRING,`userlong`STRING,`sessionid`STRING,`widgetid`STRING,`hover`STRING,`listeneradded_73d9f4ac`STRING,`listeneradded_b2157329`STRING,`trackclick`STRING,`id`STRING,`pageVersion`STRING,`companyid`STRING,`moveindate`STRING,`siteid`STRING,`buttonText`STRING,`invalid`STRING,`userlat`STRING,`pageversion`STRING\n)\nPARTITIONED BY (dt string)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n \nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo'\n``: ['CREATE DATABASE IF NOT EXISTS `sfdl_prod_drivethru`', 'DROP TABLE IF EXISTS `sfdl_prod_drivethru.websites_data_analytics_demo`', \"CREATE EXTERNAL TABLE IF NOT EXISTS\\n`sfdl_prod_drivethru.websites_data_analytics_demo`\\n(\\n`siteId`STRING,`listeneradded_56f70cc2`STRING,`ddginputtype`STRING,`name`STRING,`focus`STRING,`facilityId`STRING,`dashlanerid`STRING,`errors`STRING,`dashlanelabel`STRING,`index`STRING,`qbtmpid`STRING,`filtersused`STRING,`pageId`STRING,`listeneradded_5f3fd31e`STRING,`unitssorted`array,`event`STRING,`widgetId`STRING,`timespent`STRING,`tenantcity`STRING,`type`STRING,`discountedprice`STRING,`reservationid`STRING,`unittierid`STRING,`categoryused`STRING,`discountplanid`STRING,`unitsshown`STRING,`timestamp`STRING,`tenantpostal`STRING,`trackClick`STRING,`timeStamp`STRING,`gtmforminteractfieldid`STRING,`testid`STRING,`userLong`STRING,`imagesviewed`STRING,`hscfbound`STRING,`eventlocation`STRING,`sessionId`STRING,`immersivetranslatewalked`STRING,`sortorder`STRING,`buttontext`STRING,`facilityid`STRING,`userLat`STRING,`gramm`STRING,`discountapplied`STRING,`dashlaneclassification`STRING,`pageid`STRING,`softreservation`STRING,`category`STRING,`size`STRING,`status`STRING,`ishidden`STRING,`companyId`STRING,`discountattempted`STRING,`reservationurl`STRING,`listeneradded_d31b2f7b`STRING,`widgetname`STRING,`price`STRING,`tenantstate`STRING,`checked`STRING,`ctadumped`STRING,`nganimate`STRING,`loading`STRING,`unitgroupid`STRING,`group`STRING,`listeneradded_1ac3268d`STRING,`actiontype`STRING,`gtmforminteractid`STRING,`userlong`STRING,`sessionid`STRING,`widgetid`STRING,`hover`STRING,`listeneradded_73d9f4ac`STRING,`listeneradded_b2157329`STRING,`trackclick`STRING,`id`STRING,`pageVersion`STRING,`companyid`STRING,`moveindate`STRING,`siteid`STRING,`buttonText`STRING,`invalid`STRING,`userlat`STRING,`pageversion`STRING\\n)\\nPARTITIONED BY (dt string)\\nROW FORMAT SERDE\\n'org.openx.data.jsonserde.JsonSerDe'\\n \\nSTORED AS INPUTFORMAT\\n  'org.apache.hadoop.mapred.TextInputFormat'\\nOUTPUTFORMAT\\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\\nLOCATION\\n  's3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo'\\n\", 'MSCK REPAIR TABLE sfdl_prod_drivethru.websites_data_analytics_demo']\n: Expecting value: line 1 column 1 (char 0)"
}
    • id: 144955230
    • payload: 
      {
    "datatype_dict": null,
    "file_format": "json",
    "is_schema_datatyped": true,
    "partition": {
        "location": "s3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo/dt=2025-07-03",
        "name": "dt",
        "type": "string",
        "value": "2025-07-03"
    },
    "s3": {
        "bucket": "sfdl-drivethrujson-drivethru-prod",
        "hive_path": "s3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo",
        "key": "raw_incremental/websites_data_analytics_demo/dt=2025-07-03/events_05_00.json.gz"
    },
    "schema": {
        "name": "sfdl_prod_drivethru.websites_data_analytics_demo",
        "type": "drivethrujson"
    },
    "trace_context": {
        "parent_id": 4008600722617378712,
        "service_name": "drive-thru",
        "span_id": 3267897688359398311,
        "trace_id": 17773119154493837460
    }
}
    • running: False
    • schema_name_id: 3476
    • start_time: 2025-07-03 05:05:15
    • success: False
    • trace_id: 17773119154493837460
Partitions:
  • count: 3
  • ddl: 
    [
    "ALTER TABLE\n    `sfdl_prod_drivethru.websites_data_analytics_demo`\nADD\n\n\n PARTITION (dt='2025-08-07') location 's3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo/dt=2025-08-07'\n PARTITION (dt='2025-08-08') location 's3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo/dt=2025-08-08'\n PARTITION (dt='2025-08-12') location 's3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo/dt=2025-08-12'\n;"
]
  • name: dt
  • state: 
    [
    {
        "info": {
            "id": 1533683,
            "location": "s3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo/dt=2025-08-07",
            "name": "dt",
            "value": "2025-08-07"
        },
        "state": {
            "arrival": {
                "athena_discovery": false,
                "first_data": "2025-08-07 00:05:13",
                "last_data": "2025-08-07 23:06:47"
            },
            "parquet": {
                "first_performed": null,
                "in_progress": null,
                "job_id": null,
                "last_performed": null,
                "queued": false
            }
        }
    },
    {
        "info": {
            "id": 1533945,
            "location": "s3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo/dt=2025-08-08",
            "name": "dt",
            "value": "2025-08-08"
        },
        "state": {
            "arrival": {
                "athena_discovery": false,
                "first_data": "2025-08-08 01:05:10",
                "last_data": "2025-08-08 11:06:39"
            },
            "parquet": {
                "first_performed": null,
                "in_progress": null,
                "job_id": null,
                "last_performed": null,
                "queued": false
            }
        }
    },
    {
        "info": {
            "id": 1534922,
            "location": "s3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo/dt=2025-08-12",
            "name": "dt",
            "value": "2025-08-12"
        },
        "state": {
            "arrival": {
                "athena_discovery": false,
                "first_data": "2025-08-12 01:05:10",
                "last_data": "2025-08-12 05:06:21"
            },
            "parquet": {
                "first_performed": null,
                "in_progress": null,
                "job_id": null,
                "last_performed": null,
                "queued": false
            }
        }
    }
]
  • type: string
Schema:
  • attributes: 
    {
    "actiontype": "string",
    "buttonText": "string",
    "buttontext": "string",
    "category": "string",
    "categoryused": "string",
    "checked": "string",
    "companyId": "string",
    "companyid": "string",
    "ctadumped": "string",
    "dashlaneclassification": "string",
    "dashlanelabel": "string",
    "dashlanerid": "string",
    "ddginputtype": "string",
    "discountapplied": "string",
    "discountattempted": "string",
    "discountedprice": "string",
    "discountplanid": "string",
    "errors": "string",
    "event": "string",
    "eventlocation": "string",
    "facilityId": "string",
    "facilityid": "string",
    "filtersused": "string",
    "focus": "string",
    "gramm": "string",
    "group": "string",
    "gtmforminteractfieldid": "string",
    "gtmforminteractid": "string",
    "hover": "string",
    "hscfbound": "string",
    "id": "string",
    "imagesviewed": "string",
    "immersivetranslatewalked": "string",
    "index": "string",
    "invalid": "string",
    "ishidden": "string",
    "listeneradded_1ac3268d": "string",
    "listeneradded_56f70cc2": "string",
    "listeneradded_5f3fd31e": "string",
    "listeneradded_73d9f4ac": "string",
    "listeneradded_8f80b699": "string",
    "listeneradded_b2157329": "string",
    "listeneradded_d31b2f7b": "string",
    "loading": "string",
    "moveindate": "string",
    "name": "string",
    "nganimate": "string",
    "pageId": "string",
    "pageVersion": "string",
    "pageid": "string",
    "pageversion": "string",
    "price": "string",
    "qbtmpid": "string",
    "reservationid": "string",
    "reservationurl": "string",
    "rruidropdownitem": "string",
    "sessionId": "string",
    "sessionid": "string",
    "siteId": "string",
    "siteid": "string",
    "size": "string",
    "softreservation": "string",
    "sortorder": "string",
    "status": "string",
    "tenantcity": "string",
    "tenantpostal": "string",
    "tenantstate": "string",
    "testid": "string",
    "timeStamp": "string",
    "timespent": "string",
    "timestamp": "string",
    "trackClick": "string",
    "trackclick": "string",
    "type": "string",
    "unitgroupid": "string",
    "unitsshown": "string",
    "unitssorted": [
        "string"
    ],
    "unittierid": "string",
    "userLat": "string",
    "userLong": "string",
    "userlat": "string",
    "userlong": "string",
    "widgetId": "string",
    "widgetid": "string",
    "widgetname": "string"
}
  • created: 2025-08-04 02:05:14
  • ddl: 
    {
    "create_database": "CREATE DATABASE IF NOT EXISTS `sfdl_prod_drivethru`",
    "create_table": "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_prod_drivethru.websites_data_analytics_demo`\n(\n`siteId`STRING,`listeneradded_56f70cc2`STRING,`ddginputtype`STRING,`name`STRING,`focus`STRING,`facilityId`STRING,`dashlanerid`STRING,`errors`STRING,`dashlanelabel`STRING,`index`STRING,`qbtmpid`STRING,`filtersused`STRING,`pageId`STRING,`listeneradded_5f3fd31e`STRING,`unitssorted`array,`event`STRING,`widgetId`STRING,`timespent`STRING,`tenantcity`STRING,`type`STRING,`discountedprice`STRING,`reservationid`STRING,`unittierid`STRING,`categoryused`STRING,`discountplanid`STRING,`unitsshown`STRING,`timestamp`STRING,`tenantpostal`STRING,`trackClick`STRING,`timeStamp`STRING,`gtmforminteractfieldid`STRING,`testid`STRING,`userLong`STRING,`imagesviewed`STRING,`hscfbound`STRING,`eventlocation`STRING,`sessionId`STRING,`immersivetranslatewalked`STRING,`sortorder`STRING,`buttontext`STRING,`rruidropdownitem`STRING,`facilityid`STRING,`userLat`STRING,`gramm`STRING,`discountapplied`STRING,`dashlaneclassification`STRING,`pageid`STRING,`softreservation`STRING,`category`STRING,`size`STRING,`status`STRING,`ishidden`STRING,`companyId`STRING,`discountattempted`STRING,`reservationurl`STRING,`listeneradded_d31b2f7b`STRING,`widgetname`STRING,`price`STRING,`tenantstate`STRING,`checked`STRING,`ctadumped`STRING,`nganimate`STRING,`loading`STRING,`unitgroupid`STRING,`listeneradded_8f80b699`STRING,`group`STRING,`listeneradded_1ac3268d`STRING,`actiontype`STRING,`listeneradded_b2157329`STRING,`gtmforminteractid`STRING,`userlong`STRING,`sessionid`STRING,`hover`STRING,`listeneradded_73d9f4ac`STRING,`widgetid`STRING,`trackclick`STRING,`id`STRING,`pageVersion`STRING,`companyid`STRING,`moveindate`STRING,`siteid`STRING,`buttonText`STRING,`invalid`STRING,`userlat`STRING,`pageversion`STRING\n)\nPARTITIONED BY (dt string)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n \nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo'\n",
    "drop_table": "DROP TABLE IF EXISTS `sfdl_prod_drivethru.websites_data_analytics_demo`",
    "repair_table": "MSCK REPAIR TABLE sfdl_prod_drivethru.websites_data_analytics_demo"
}
Server:
  • hostname: schemamanager-5475496c57-7dpqb
  • name: schema_manager
  • process_start: 2026-01-09 11:27:47
  • sf_env: prod
  • version: 2112174301
Raw: 
{
    "name": {
        "created": "2025-02-28 14:05:11",
        "disabled": false,
        "format": "raw",
        "id": 3476,
        "is_partitioned": true,
        "is_schema_datatyped": true,
        "name": {
            "database": "sfdl_prod_drivethru",
            "table": "websites_data_analytics_demo"
        },
        "type": "drivethrujson"
    },
    "partition": {
        "count": 3,
        "ddl": [
            "ALTER TABLE\n    `sfdl_prod_drivethru.websites_data_analytics_demo`\nADD\n\n\n PARTITION (dt='2025-08-07') location 's3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo/dt=2025-08-07'\n PARTITION (dt='2025-08-08') location 's3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo/dt=2025-08-08'\n PARTITION (dt='2025-08-12') location 's3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo/dt=2025-08-12'\n;"
        ],
        "name": "dt",
        "state": [
            {
                "info": {
                    "id": 1533683,
                    "location": "s3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo/dt=2025-08-07",
                    "name": "dt",
                    "value": "2025-08-07"
                },
                "state": {
                    "arrival": {
                        "athena_discovery": false,
                        "first_data": "2025-08-07 00:05:13",
                        "last_data": "2025-08-07 23:06:47"
                    },
                    "parquet": {
                        "first_performed": null,
                        "in_progress": null,
                        "job_id": null,
                        "last_performed": null,
                        "queued": false
                    }
                }
            },
            {
                "info": {
                    "id": 1533945,
                    "location": "s3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo/dt=2025-08-08",
                    "name": "dt",
                    "value": "2025-08-08"
                },
                "state": {
                    "arrival": {
                        "athena_discovery": false,
                        "first_data": "2025-08-08 01:05:10",
                        "last_data": "2025-08-08 11:06:39"
                    },
                    "parquet": {
                        "first_performed": null,
                        "in_progress": null,
                        "job_id": null,
                        "last_performed": null,
                        "queued": false
                    }
                }
            },
            {
                "info": {
                    "id": 1534922,
                    "location": "s3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo/dt=2025-08-12",
                    "name": "dt",
                    "value": "2025-08-12"
                },
                "state": {
                    "arrival": {
                        "athena_discovery": false,
                        "first_data": "2025-08-12 01:05:10",
                        "last_data": "2025-08-12 05:06:21"
                    },
                    "parquet": {
                        "first_performed": null,
                        "in_progress": null,
                        "job_id": null,
                        "last_performed": null,
                        "queued": false
                    }
                }
            }
        ],
        "type": "string"
    },
    "schema": {
        "attributes": {
            "actiontype": "string",
            "buttonText": "string",
            "buttontext": "string",
            "category": "string",
            "categoryused": "string",
            "checked": "string",
            "companyId": "string",
            "companyid": "string",
            "ctadumped": "string",
            "dashlaneclassification": "string",
            "dashlanelabel": "string",
            "dashlanerid": "string",
            "ddginputtype": "string",
            "discountapplied": "string",
            "discountattempted": "string",
            "discountedprice": "string",
            "discountplanid": "string",
            "errors": "string",
            "event": "string",
            "eventlocation": "string",
            "facilityId": "string",
            "facilityid": "string",
            "filtersused": "string",
            "focus": "string",
            "gramm": "string",
            "group": "string",
            "gtmforminteractfieldid": "string",
            "gtmforminteractid": "string",
            "hover": "string",
            "hscfbound": "string",
            "id": "string",
            "imagesviewed": "string",
            "immersivetranslatewalked": "string",
            "index": "string",
            "invalid": "string",
            "ishidden": "string",
            "listeneradded_1ac3268d": "string",
            "listeneradded_56f70cc2": "string",
            "listeneradded_5f3fd31e": "string",
            "listeneradded_73d9f4ac": "string",
            "listeneradded_8f80b699": "string",
            "listeneradded_b2157329": "string",
            "listeneradded_d31b2f7b": "string",
            "loading": "string",
            "moveindate": "string",
            "name": "string",
            "nganimate": "string",
            "pageId": "string",
            "pageVersion": "string",
            "pageid": "string",
            "pageversion": "string",
            "price": "string",
            "qbtmpid": "string",
            "reservationid": "string",
            "reservationurl": "string",
            "rruidropdownitem": "string",
            "sessionId": "string",
            "sessionid": "string",
            "siteId": "string",
            "siteid": "string",
            "size": "string",
            "softreservation": "string",
            "sortorder": "string",
            "status": "string",
            "tenantcity": "string",
            "tenantpostal": "string",
            "tenantstate": "string",
            "testid": "string",
            "timeStamp": "string",
            "timespent": "string",
            "timestamp": "string",
            "trackClick": "string",
            "trackclick": "string",
            "type": "string",
            "unitgroupid": "string",
            "unitsshown": "string",
            "unitssorted": [
                "string"
            ],
            "unittierid": "string",
            "userLat": "string",
            "userLong": "string",
            "userlat": "string",
            "userlong": "string",
            "widgetId": "string",
            "widgetid": "string",
            "widgetname": "string"
        },
        "created": "2025-08-04 02:05:14",
        "ddl": {
            "create_database": "CREATE DATABASE IF NOT EXISTS `sfdl_prod_drivethru`",
            "create_table": "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_prod_drivethru.websites_data_analytics_demo`\n(\n`siteId`STRING,`listeneradded_56f70cc2`STRING,`ddginputtype`STRING,`name`STRING,`focus`STRING,`facilityId`STRING,`dashlanerid`STRING,`errors`STRING,`dashlanelabel`STRING,`index`STRING,`qbtmpid`STRING,`filtersused`STRING,`pageId`STRING,`listeneradded_5f3fd31e`STRING,`unitssorted`array,`event`STRING,`widgetId`STRING,`timespent`STRING,`tenantcity`STRING,`type`STRING,`discountedprice`STRING,`reservationid`STRING,`unittierid`STRING,`categoryused`STRING,`discountplanid`STRING,`unitsshown`STRING,`timestamp`STRING,`tenantpostal`STRING,`trackClick`STRING,`timeStamp`STRING,`gtmforminteractfieldid`STRING,`testid`STRING,`userLong`STRING,`imagesviewed`STRING,`hscfbound`STRING,`eventlocation`STRING,`sessionId`STRING,`immersivetranslatewalked`STRING,`sortorder`STRING,`buttontext`STRING,`rruidropdownitem`STRING,`facilityid`STRING,`userLat`STRING,`gramm`STRING,`discountapplied`STRING,`dashlaneclassification`STRING,`pageid`STRING,`softreservation`STRING,`category`STRING,`size`STRING,`status`STRING,`ishidden`STRING,`companyId`STRING,`discountattempted`STRING,`reservationurl`STRING,`listeneradded_d31b2f7b`STRING,`widgetname`STRING,`price`STRING,`tenantstate`STRING,`checked`STRING,`ctadumped`STRING,`nganimate`STRING,`loading`STRING,`unitgroupid`STRING,`listeneradded_8f80b699`STRING,`group`STRING,`listeneradded_1ac3268d`STRING,`actiontype`STRING,`listeneradded_b2157329`STRING,`gtmforminteractid`STRING,`userlong`STRING,`sessionid`STRING,`hover`STRING,`listeneradded_73d9f4ac`STRING,`widgetid`STRING,`trackclick`STRING,`id`STRING,`pageVersion`STRING,`companyid`STRING,`moveindate`STRING,`siteid`STRING,`buttonText`STRING,`invalid`STRING,`userlat`STRING,`pageversion`STRING\n)\nPARTITIONED BY (dt string)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n \nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrujson-drivethru-prod/raw_incremental/websites_data_analytics_demo'\n",
            "drop_table": "DROP TABLE IF EXISTS `sfdl_prod_drivethru.websites_data_analytics_demo`",
            "repair_table": "MSCK REPAIR TABLE sfdl_prod_drivethru.websites_data_analytics_demo"
        }
    },
    "server": {
        "hostname": "schemamanager-5475496c57-7dpqb",
        "name": "schema_manager",
        "process_start": "2026-01-09 11:27:47",
        "sf_env": "prod",
        "version": "2112174301"
    }
}
©2019 SpareFoot Data Platform