Skip to content

Instantly share code, notes, and snippets.

Show Gist options
  • Save oatsandsugar/d1201bab9c7a77fc0fa547aa4d9b3048 to your computer and use it in GitHub Desktop.
Save oatsandsugar/d1201bab9c7a77fc0fa547aa4d9b3048 to your computer and use it in GitHub Desktop.
{'id': '0a275942-a069-48bd-9f70-07168334a149',
'name': 'US Telemetry classification pipeline',
'streams': [{'id': '35214455-ef86-48c6-8943-c1991ce34933',
'isReady': True,
'stream': {'source': {'type': 'GCS',
'id': '4d034c34-4976-423e-b93a-379ec8645873',
'bucket': 'dtl-handset-telemetry',
'fileName': 'us_340M.csv',
'format': 'Csv',
'params': {}},
'pipelines': [{'transformations': [{'type': 'Structure',
'outputSchema': [{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Full Name'],
'class': 'Full Name',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['First Name'],
'class': 'First Name',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Family Name'],
'class': 'Family Name',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Subscriber Company'],
'class': 'Subscriber Company',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Account Identifier'],
'class': 'Account Identifier',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Subscriber Street'],
'class': 'Subscriber Street',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Subscriber City'],
'class': 'Subscriber City',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Subscriber Zip'],
'class': 'Subscriber Zip',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Subscriber Phone Number'],
'class': 'Subscriber Phone Number',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Subscriber Email'],
'class': 'Subscriber Email',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Mac Address'],
'class': 'Mac Address',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['IMEI'],
'class': 'IMEI',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Latitude'],
'class': 'Latitude',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Longitude'],
'class': 'Longitude',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Altitude'],
'class': 'Altitude',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Geo Hash'],
'class': 'Geo Hash',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['IP Address'],
'class': 'IP Address',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Mobile Network Code'],
'class': 'Mobile Network Code',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Network Provider'],
'class': 'Network Provider',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Manufacturer'],
'class': 'Manufacturer',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Device ID'],
'class': 'Device ID',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['OS'],
'class': 'OS',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Device Model'],
'class': 'Device Model',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Screen Resolution Width'],
'class': 'Screen Resolution Width',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Screen Resolution Height'],
'class': 'Screen Resolution Height',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Device Memory'],
'class': 'Device Memory',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Device Storage'],
'class': 'Device Storage',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Device Language'],
'class': 'Device Language',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Device System Uptime'],
'class': 'Device System Uptime',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Device Used Storage'],
'class': 'Device Used Storage',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Device Unused Storage'],
'class': 'Device Unused Storage',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Device Used Memory'],
'class': 'Device Used Memory',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Device Unused Memory'],
'class': 'Device Unused Memory',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Device CPU Usage'],
'class': 'Device CPU Usage',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Device Battery Usage'],
'class': 'Device Battery Usage',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Device Battery State'],
'class': 'Device Battery State',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Device Horizontal Accuracy'],
'class': 'Device Horizontal Accuracy',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Device Vertical Accuracy'],
'class': 'Device Vertical Accuracy',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Device Velocity Speed'],
'class': 'Device Velocity Speed',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Device Velocity Bearing'],
'class': 'Device Velocity Bearing',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Session State or Territory'],
'class': 'Session State or Territory',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Session Country'],
'class': 'Session Country',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Session Connection Type'],
'class': 'Session Connection Type',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Session Connection Technology'],
'class': 'Session Connection Technology',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Session Start'],
'class': 'Session Start',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Session End'],
'class': 'Session End',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Session Timezone'],
'class': 'Session Timezone',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['QoS Timestamp'],
'class': 'QoS Timestamp',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Session Delta Transmitted Bytes'],
'class': 'Session Delta Transmitted Bytes',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Session Delta Received Bytes'],
'class': 'Session Delta Received Bytes',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Mobile Connection Generation'],
'class': 'Mobile Connection Generation',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Mobile Channel'],
'class': 'Mobile Channel',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Mobile Country Code'],
'class': 'Mobile Country Code',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Base Station Identity Code'],
'class': 'Base Station Identity Code',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Physical Cell Identifier'],
'class': 'Physical Cell Identifier',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Reference Signal Received Power'],
'class': 'Reference Signal Received Power',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Reference Signal Received Quality'],
'class': 'Reference Signal Received Quality',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Reference Signal Signal to Noise Ratio'],
'class': 'Reference Signal Signal to Noise Ratio',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Channel Quality Indicator'],
'class': 'Channel Quality Indicator',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Timing Advance'],
'class': 'Timing Advance',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['SSID'],
'class': 'SSID',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['WiFi Channel'],
'class': 'WiFi Channel',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['WiFi Encryption'],
'class': 'WiFi Encryption',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Modulation and Coding Scheme'],
'class': 'Modulation and Coding Scheme',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['BSSID'],
'class': 'BSSID',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Wifi Frequency'],
'class': 'Wifi Frequency',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Upload Throughput'],
'class': 'Upload Throughput',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Download Throughput'],
'class': 'Download Throughput',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Latency Average'],
'class': 'Latency Average',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Link Speed'],
'class': 'Link Speed',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Signal Strength'],
'class': 'Signal Strength',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Jitter'],
'class': 'Jitter',
'pickStrategy': 'HighScore'},
{'type': 'Class',
'outputDataType': 'String',
'outputPath': ['Packet Loss'],
'class': 'Packet Loss',
'pickStrategy': 'HighScore'}]}],
'pipelines': [],
'target': {'type': 'GCS',
'id': '5de82ca8-da4f-4355-a29b-8733b0193f66',
'bucket': 'dtl-handset-telemetry',
'fileName': 'cleaned_US_handset_telemetry_data.csv',
'format': 'Csv',
'params': {}}}]}}]}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment