Skip to content

Instantly share code, notes, and snippets.

@wenhoujx
Created July 6, 2023 16:18
Show Gist options
  • Save wenhoujx/7ac9668f627e2c3cdaba6bb6d08b3563 to your computer and use it in GitHub Desktop.
Save wenhoujx/7ac9668f627e2c3cdaba6bb6d08b3563 to your computer and use it in GitHub Desktop.
fds-map.json
{
"in_connections": {
"dtrip_fermate_anag": {
"type": "local_file",
"filename": "/Users/wenshuaihou/code/ct-data/fds/dtrip_fermate_anag.jsonl"
},
"dtrip_fermate": {
"type": "local_file",
"filename": "/Users/wenshuaihou/code/ct-data/fds/small/dtrip_fermate.jsonl"
},
"dtrip_percorsi": {
"type": "local_file",
"filename": "/Users/wenshuaihou/code/ct-data/fds/dtrip_percorsi.jsonl"
},
"dtrip_linee": {
"type": "local_file",
"filename": "/Users/wenshuaihou/code/ct-data/fds/dtrip_linee.jsonl"
},
"dtrip_corse_val": {
"type": "local_file",
"filename": "/Users/wenshuaihou/code/ct-data/fds/small/dtrip_corse_val.jsonl"
}
},
"in_encodings": {
"dtrip_fermate_anag": {
"type": "read_documents",
"input": "dtrip_fermate_anag",
"mime_type": "application/x-ndjson"
},
"dtrip_fermate": {
"type": "read_documents",
"input": "dtrip_fermate",
"mime_type": "application/x-ndjson"
},
"dtrip_percorsi": {
"type": "read_documents",
"input": "dtrip_percorsi",
"mime_type": "application/x-ndjson"
},
"dtrip_linee": {
"type": "read_documents",
"input": "dtrip_linee",
"mime_type": "application/x-ndjson"
},
"dtrip_corse_val": {
"type": "read_documents",
"input": "dtrip_corse_val",
"mime_type": "application/x-ndjson"
}
},
"in_concepts": {
"concepttransitstop": {
"source_iterator": "dtrip_fermate_anagjsonl_root",
"concept_type": "concept.transit.stop"
},
"concepttransitscheduled_trip": {
"source_iterator": "dtrip_percorsi_root",
"concept_type": "concept.transit.path"
},
"concepttransitroute": {
"source_iterator": "dtrip_percorsi_root",
"concept_type": "concept.transit.route"
},
"8e395c96-55db-4fa3-8150-69b5524103a3": {
"source_iterator": "dtrip_fermate_root",
"concept_type": "concept.transit.stop"
},
"concepttransitstop_time": {
"source_iterator": "dtrip_fermate_root",
"concept_type": "concept._.belongs_to"
},
"3d9ef1d9-a278-46db-9d72-f9d9b622d423": {
"source_iterator": "dtrip_fermate_root",
"concept_type": "concept.transit.path"
},
"22fe4ea3-0621-409b-9ff1-b728dd96867e": {
"source_iterator": "dtrip_linee_root",
"concept_type": "concept.transit.route"
},
"90d7c191-7e1d-438a-b8f4-24c8bb977f8f": {
"source_iterator": "dtrip_corse_val",
"concept_type": "concept.transit.scheduled_trip"
},
"concept__path": {
"source_iterator": "dtrip_corse_val",
"concept_type": "concept.transit.path"
}
},
"in_iterators": {
"dtrip_fermate_anagjsonl_root": {
"path": [],
"collection_type": "single",
"dataset": "dtrip_fermate_anag"
},
"dtrip_fermate_root": {
"path": [],
"collection_type": "single",
"dataset": "dtrip_fermate"
},
"dtrip_percorsi_root": {
"path": [],
"collection_type": "single",
"dataset": "dtrip_percorsi"
},
"dtrip_linee_root": {
"path": [],
"collection_type": "single",
"dataset": "dtrip_linee"
},
"dtrip_corse_val": {
"path": [],
"collection_type": "single",
"dataset": "dtrip_corse_val"
}
},
"in_properties": {
"dtrip_fermate_anagjsonl_root-record-concept-0_12": {
"on_concept": "concepttransitstop",
"property_type": "property.geo.geopoint",
"values": {
"LATITUDINE-normalized": {
"location": "property",
"property_type": "LATITUDINE-normalized"
},
"LONGITUDINE-normalized": {
"location": "property",
"property_type": "LONGITUDINE-normalized"
}
}
},
"dtrip_fermate_anagjsonl_root-record-concept-0_14": {
"on_concept": "concepttransitstop",
"property_type": "property.txt.name",
"values": {
"value": {
"location": "property",
"property_type": "NOME"
}
}
},
"dtrip_fermate_anagjsonl_root-record-concept-0_15": {
"on_concept": "concepttransitstop",
"property_type": "property.txt.abbreviation",
"values": {
"value": {
"location": "property",
"property_type": "NOMEBREVE"
}
}
},
"dtrip_fermate_anagjsonl_root-record-concept-0_17": {
"on_concept": "concepttransitstop",
"property_type": "property._.id",
"values": {
"value": {
"location": "property",
"property_type": "PALINA"
}
}
},
"concepttransitscheduled_trip-property_descriptor": {
"on_concept": "concepttransitscheduled_trip",
"property_type": "property.txt.description",
"values": {
"value": {
"location": "property",
"property_type": "DESCRIZIONE"
}
}
},
"f4cf15cb-fbad-42e0-a59d-272ecebbd43b": {
"on_concept": "concepttransitscheduled_trip",
"property_type": "property.transit.reverse_direction",
"values": {
"value": {
"location": "property",
"property_type": "VERSO-final"
}
}
},
"concepttransitroute-property_descriptor": {
"on_concept": "concepttransitroute",
"property_type": "property._.id",
"values": {
"value": {
"location": "property",
"property_type": "LINEA"
}
}
},
"4a852466-8a7d-4d0d-89b9-0aaaa55bce06": {
"on_concept": "concepttransitscheduled_trip",
"property_type": "property.txt.abbreviation",
"values": {
"value": {
"location": "property",
"property_type": "PERCORSO"
}
}
},
"8e395c96-55db-4fa3-8150-69b5524103a3-property_descriptor": {
"on_concept": "8e395c96-55db-4fa3-8150-69b5524103a3",
"property_type": "property._.id",
"values": {
"value": {
"location": "property",
"property_type": "PALINA"
}
}
},
"concepttransitstop_time-property_descriptor": {
"on_concept": "concepttransitstop_time",
"property_type": "property.num.sequence",
"values": {
"value": {
"location": "property",
"property_type": "NORDINE"
}
}
},
"3d9ef1d9-a278-46db-9d72-f9d9b622d423-property_descriptor": {
"on_concept": "3d9ef1d9-a278-46db-9d72-f9d9b622d423",
"property_type": "property._.id",
"values": {
"value": {
"location": "property",
"property_type": "LINEA"
}
}
},
"d58a6661-f9fa-46ef-afe7-b47359b63096": {
"on_concept": "3d9ef1d9-a278-46db-9d72-f9d9b622d423",
"property_type": "property.txt.abbreviation",
"values": {
"value": {
"location": "property",
"property_type": "PERCORSO"
}
}
},
"2593ab24-a2f0-440a-87c1-3dbe0829e16c": {
"on_concept": "3d9ef1d9-a278-46db-9d72-f9d9b622d423",
"property_type": "property.transit.reverse_direction",
"values": {
"VERSO-final": {
"location": "property",
"property_type": "VERSO-final"
}
}
},
"f7f51d84-715a-4e0e-8c4a-7cd436844a7e": {
"on_concept": "concepttransitscheduled_trip",
"property_type": "property._.id",
"values": {
"value": {
"location": "property",
"property_type": "LINEA"
}
}
},
"22fe4ea3-0621-409b-9ff1-b728dd96867e-property_descriptor": {
"on_concept": "22fe4ea3-0621-409b-9ff1-b728dd96867e",
"property_type": "property._.id",
"values": {
"value": {
"location": "property",
"property_type": "LINEA"
}
}
},
"f87f58bc-7024-4f19-a5e7-55ad9dd87183": {
"on_concept": "22fe4ea3-0621-409b-9ff1-b728dd96867e",
"property_type": "property.txt.name",
"values": {
"value": {
"location": "property",
"property_type": "NOMEESTESO"
}
}
},
"93f95121-8d74-4590-a7a5-0cc3838aa784-property_descriptor": {
"on_concept": "concept__path",
"property_type": "property._.id",
"values": {
"value": {
"location": "property",
"property_type": "LINEA"
}
}
},
"c8fbe730-cadc-473b-87c8-d95c81ff6c6f": {
"on_concept": "concept__path",
"property_type": "property.transit.reverse_direction",
"values": {
"VERSO-final": {
"location": "property",
"property_type": "VERSO-final"
}
}
},
"90d7c191-7e1d-438a-b8f4-24c8bb977f8f-property_descriptor": {
"on_concept": "concept__path",
"property_type": "property.txt.abbreviation",
"values": {
"value": {
"location": "property",
"property_type": "PERCORSO"
}
}
},
"4374fd58-257e-4f15-86ac-1609445d25f5": {
"on_concept": "90d7c191-7e1d-438a-b8f4-24c8bb977f8f",
"property_type": "property.transit.departure_time",
"values": {
"value": {
"location": "property",
"property_type": "ORA_INIZIO"
}
}
},
"58c29f28-36b2-4ddc-9938-635de88d961a": {
"on_concept": "90d7c191-7e1d-438a-b8f4-24c8bb977f8f",
"property_type": "property.transit.arrival_time",
"values": {
"value": {
"location": "property",
"property_type": "ORA_FINE"
}
}
}
},
"adhoc_properties": {},
"in_value_sets": {
"dtrip_fermate_anagjsonl_root_values": {
"iterator": "dtrip_fermate_anagjsonl_root",
"format": "key_value"
},
"dtrip_fermate_root_values": {
"iterator": "dtrip_fermate_root",
"format": "key_value"
},
"dtrip_percorsi_root_values": {
"iterator": "dtrip_percorsi_root",
"format": "key_value"
},
"dtrip_linee_root_values": {
"iterator": "dtrip_linee_root",
"format": "key_value"
},
"dtrip_corse_val_root_values": {
"iterator": "dtrip_corse_val",
"format": "key_value"
}
},
"in_links": {
"link_related_to-concepttransitscheduled_trip-concepttransitroute": {
"link_type": "link._.related_to",
"from_concept": "concepttransitscheduled_trip",
"to_concept": "concepttransitroute"
},
"link_related_to-8e395c96-55db-4fa3-8150-69b5524103a3-concepttransitstop_time": {
"link_type": "link._.as_a",
"from_concept": "8e395c96-55db-4fa3-8150-69b5524103a3",
"to_concept": "concepttransitstop_time"
},
"link_related_to-concepttransitstop_time-3d9ef1d9-a278-46db-9d72-f9d9b622d423": {
"link_type": "link._.related_to",
"from_concept": "concepttransitstop_time",
"to_concept": "3d9ef1d9-a278-46db-9d72-f9d9b622d423"
},
"link_related_to-90d7c191-7e1d-438a-b8f4-24c8bb977f8f-concept__path": {
"link_type": "link._.related_to",
"from_concept": "90d7c191-7e1d-438a-b8f4-24c8bb977f8f",
"to_concept": "concept__path"
}
},
"in_property_parsers": {
"dtrip_fermate_anagjsonl_root-record-concept-0_14": {
"on_property": "dtrip_fermate_anagjsonl_root-record-concept-0_14",
"params": {
"value": "value"
}
},
"dtrip_fermate_anagjsonl_root-record-concept-0_15": {
"on_property": "dtrip_fermate_anagjsonl_root-record-concept-0_15",
"params": {
"value": "value"
}
},
"dtrip_fermate_anagjsonl_root-record-concept-0_17": {
"on_property": "dtrip_fermate_anagjsonl_root-record-concept-0_17",
"params": {
"value": "value"
}
},
"parse-dtrip_fermate_anagjsonl_root-record-concept-0_12": {
"on_property": "dtrip_fermate_anagjsonl_root-record-concept-0_12",
"parser_type": "parser.geo.utm_32n_to_geopoint",
"params": {
"y": "LATITUDINE-normalized",
"x": "LONGITUDINE-normalized"
}
},
"parse-concepttransitscheduled_trip-property_descriptor": {
"on_property": "concepttransitscheduled_trip-property_descriptor",
"params": {
"value": "value"
}
},
"parse-f4cf15cb-fbad-42e0-a59d-272ecebbd43b": {
"on_property": "f4cf15cb-fbad-42e0-a59d-272ecebbd43b",
"parser_type": "parser._.string_to_bool",
"params": {
"value": "value"
}
},
"parse-concepttransitroute-property_descriptor": {
"on_property": "concepttransitroute-property_descriptor",
"params": {
"value": "value"
}
},
"parse-4a852466-8a7d-4d0d-89b9-0aaaa55bce06": {
"on_property": "4a852466-8a7d-4d0d-89b9-0aaaa55bce06",
"params": {
"value": "value"
}
},
"parse-8e395c96-55db-4fa3-8150-69b5524103a3-property_descriptor": {
"on_property": "8e395c96-55db-4fa3-8150-69b5524103a3-property_descriptor",
"params": {
"value": "value"
}
},
"parse-concepttransitstop_time-property_descriptor": {
"on_property": "concepttransitstop_time-property_descriptor",
"parser_type": "parser.num.string_to_integer",
"params": {
"value": "value"
}
},
"parse-d58a6661-f9fa-46ef-afe7-b47359b63096": {
"on_property": "d58a6661-f9fa-46ef-afe7-b47359b63096",
"params": {
"value": "value"
}
},
"parse-3d9ef1d9-a278-46db-9d72-f9d9b622d423-property_descriptor": {
"on_property": "3d9ef1d9-a278-46db-9d72-f9d9b622d423-property_descriptor",
"params": {
"value": "value"
}
},
"parse-2593ab24-a2f0-440a-87c1-3dbe0829e16c": {
"on_property": "2593ab24-a2f0-440a-87c1-3dbe0829e16c",
"parser_type": "parser._.string_to_bool",
"params": {
"value": "VERSO-final"
}
},
"parse-f7f51d84-715a-4e0e-8c4a-7cd436844a7e": {
"on_property": "f7f51d84-715a-4e0e-8c4a-7cd436844a7e",
"params": {
"value": "value"
}
},
"parse-22fe4ea3-0621-409b-9ff1-b728dd96867e-property_descriptor": {
"on_property": "22fe4ea3-0621-409b-9ff1-b728dd96867e-property_descriptor",
"params": {
"value": "value"
}
},
"parse-f87f58bc-7024-4f19-a5e7-55ad9dd87183": {
"on_property": "f87f58bc-7024-4f19-a5e7-55ad9dd87183",
"params": {
"value": "value"
}
},
"parse-93f95121-8d74-4590-a7a5-0cc3838aa784-property_descriptor": {
"on_property": "93f95121-8d74-4590-a7a5-0cc3838aa784-property_descriptor",
"params": {
"value": "value"
}
},
"parse-90d7c191-7e1d-438a-b8f4-24c8bb977f8f-property_descriptor": {
"on_property": "90d7c191-7e1d-438a-b8f4-24c8bb977f8f-property_descriptor",
"params": {
"value": "value"
}
},
"parse-c8fbe730-cadc-473b-87c8-d95c81ff6c6f": {
"on_property": "c8fbe730-cadc-473b-87c8-d95c81ff6c6f",
"parser_type": "parser._.string_to_bool",
"params": {
"value": "VERSO-final"
}
},
"parse-4374fd58-257e-4f15-86ac-1609445d25f5": {
"on_property": "4374fd58-257e-4f15-86ac-1609445d25f5",
"parser_type": "parser.cal.seconds_to_duration",
"params": {
"value": "value"
}
},
"parse-58c29f28-36b2-4ddc-9938-635de88d961a": {
"on_property": "58c29f28-36b2-4ddc-9938-635de88d961a",
"parser_type": "parser.cal.seconds_to_duration",
"params": {
"value": "value"
}
}
},
"in_transforms": {
"str_replace-longitudine": {
"dataset": "dtrip_fermate_anag",
"from_path": [
"LONGITUDINE"
],
"patterns": [
","
],
"replace_with": ".",
"type": "str_replace",
"to_path": [
"LONGITUDINE-normalized"
]
},
"str_replace-latitudine": {
"dataset": "dtrip_fermate_anag",
"from_path": [
"LATITUDINE"
],
"patterns": [
","
],
"replace_with": ".",
"type": "str_replace",
"to_path": [
"LATITUDINE-normalized"
]
},
"str_replace-verso": {
"dataset": "dtrip_percorsi",
"from_path": [
"VERSO"
],
"patterns": [
"As"
],
"replace_with": "0",
"type": "str_replace",
"to_path": [
"VERSO-str_replace"
]
},
"str_replace-verso-str_replace": {
"dataset": "dtrip_percorsi",
"from_path": [
"VERSO-str_replace"
],
"depends_on": [
"str_replace-verso"
],
"patterns": [
"Di"
],
"replace_with": "1",
"type": "str_replace",
"to_path": [
"VERSO-final"
]
},
"47e801b5-ce37-44c9-a021-add1b84c6666": {
"dataset": "dtrip_fermate",
"from_path": [
"VERSO"
],
"patterns": [
"As"
],
"replace_with": "0",
"type": "str_replace",
"to_path": [
"VERSO-str_replace"
]
},
"e3058c7a-a35e-431e-9bf2-eac4eff5d2a0": {
"dataset": "dtrip_fermate",
"from_path": [
"VERSO-str_replace"
],
"depends_on": [
"47e801b5-ce37-44c9-a021-add1b84c6666"
],
"patterns": [
"Di"
],
"replace_with": "1",
"type": "str_replace",
"to_path": [
"VERSO-final"
]
},
"76dc9646-a306-41ac-96f1-b7ed08791242": {
"dataset": "dtrip_corse_val",
"from_path": [
"VERSO"
],
"patterns": [
"As"
],
"replace_with": "0",
"type": "str_replace",
"to_path": [
"VERSO-str_replace"
]
},
"9cc74239-4d9d-40c2-afd4-eb40f0667734": {
"dataset": "dtrip_corse_val",
"from_path": [
"VERSO-str_replace"
],
"depends_on": [
"76dc9646-a306-41ac-96f1-b7ed08791242"
],
"patterns": [
"Di"
],
"replace_with": "1",
"type": "str_replace",
"to_path": [
"VERSO-final"
]
}
},
"out_documents": {
"stops": {
"concept_type": "concept.transit.stop"
},
"stop_times": {
"concept_type": "concept.transit.stop_time"
},
"trips": {
"concept_type": "concept.transit.scheduled_trip"
},
"routes": {
"concept_type": "concept.transit.route"
}
},
"out_fields": {
"name": {
"type": "property_value",
"on_document": "stops",
"document_path": [
"stop_name"
],
"property_type": "property.txt.name",
"graph_path": []
},
"abbreviation": {
"type": "property_value",
"on_document": "stops",
"document_path": [
"stop_code"
],
"property_type": "property.txt.abbreviation",
"graph_path": []
},
"id": {
"type": "property_value",
"on_document": "stops",
"document_path": [
"stop_id"
],
"property_type": "property._.id",
"graph_path": []
},
"geolocation_lat": {
"type": "property_value",
"on_document": "stops",
"document_path": [
"stop_lat"
],
"property_type": "property.geo.geopoint",
"graph_path": [],
"generator": "generator.geo.geopoint_to_floats",
"generator_output": "lat"
},
"geolocation_lon": {
"type": "property_value",
"on_document": "stops",
"document_path": [
"stop_lon"
],
"property_type": "property.geo.geopoint",
"graph_path": [],
"generator": "generator.geo.geopoint_to_floats",
"generator_output": "lon"
},
"scheduledtripid": {
"type": "concept_reference",
"on_document": "stop_times",
"document_path": [
"trip_id"
],
"graph_path": [
"link._.related_to",
"concept.transit.scheduled_trip"
]
},
"88948e03-d470-4660-b4df-b5a913c7f3f3": {
"type": "concept_reference",
"on_document": "trips",
"document_path": [
"trip_id"
],
"graph_path": []
},
"f7ffa93e-b6f3-4ff7-bcc2-1d178d241749": {
"type": "property_value",
"on_document": "routes",
"document_path": [
"route_id"
],
"property_type": "property._.id",
"graph_path": []
},
"route_short_name": {
"type": "property_value",
"on_document": "routes",
"document_path": [
"route_short_name"
],
"property_type": "property._.id",
"graph_path": []
},
"transitroutename": {
"type": "property_value",
"on_document": "routes",
"document_path": [
"route_long_name"
],
"property_type": "property.txt.name",
"graph_path": []
},
"departuretime": {
"type": "property_value",
"on_document": "trips",
"document_path": [
"Departure Time"
],
"property_type": "property.transit.departure_time",
"graph_path": [],
"generator": "generator.cal.duration_to_hhmmss",
"generator_output": "value"
},
"arrivaltime": {
"type": "property_value",
"on_document": "trips",
"document_path": [
"Arrival Time"
],
"property_type": "property.transit.arrival_time",
"generator": "generator.cal.duration_to_hhmmss",
"generator_output": "value"
},
"description": {
"type": "property_value",
"on_document": "trips",
"document_path": [
"Description"
],
"property_type": "property.txt.description",
"graph_path": [
"link._.related_to",
"concept.transit.path"
]
},
"reversedirection": {
"type": "property_value",
"on_document": "trips",
"document_path": [
"direction_id"
],
"property_type": "property.transit.reverse_direction",
"graph_path": [
"link._.related_to",
"concept.transit.path"
],
"generator": "generator._.bool_to_integer",
"generator_output": "value"
},
"pathabbreviation": {
"type": "property_value",
"on_document": "trips",
"document_path": [
"trip_short_name"
],
"property_type": "property.txt.abbreviation",
"graph_path": [
"link._.related_to",
"concept.transit.path"
]
},
"stop_id": {
"type": "property_value",
"on_document": "stop_times",
"document_path": [
"stop_id"
],
"property_type": "property._.id",
"graph_path": [
"link._.related_to",
"concept._.belongs_to",
"link._.as_a",
"concept.transit.stop"
]
},
"stop_sequence": {
"type": "property_value",
"on_document": "stop_times",
"document_path": [
"stop_sequence"
],
"property_type": "property.num.sequence",
"graph_path": [
"link._.related_to",
"concept._.belongs_to"
]
},
"transitstoptimearrivaltime": {
"type": "property_value",
"on_document": "stop_times",
"document_path": [
"arrival_time"
],
"property_type": "property.transit.arrival_time",
"graph_path": [],
"generator": "generator.cal.duration_to_hhmmss",
"generator_output": "value"
},
"transitstoptimedeparturetime": {
"type": "property_value",
"on_document": "stop_times",
"document_path": [
"departure_time"
],
"property_type": "property.transit.departure_time",
"graph_path": [],
"generator": "generator.cal.duration_to_hhmmss",
"generator_output": "value"
}
},
"resolutions": {
"concepttransitscheduled_trip--property_id": {
"type": "concept.transit.path",
"on": [
"property._.id",
"property.txt.abbreviation",
"property.transit.reverse_direction"
]
},
"concepttransitroute--property_id": {
"type": "concept.transit.route",
"on": [
"property._.id"
]
},
"stop-id": {
"type": "concept.transit.stop",
"on": [
"property._.id"
]
}
},
"out_encodings": {
"stops": {
"type": "emit_table_lines",
"projection": "stops",
"mime_type": "text/csv"
},
"stop_times": {
"type": "emit_table_lines",
"projection": "stop_times",
"mime_type": "text/csv"
},
"trips": {
"type": "emit_table_lines",
"projection": "trips",
"mime_type": "text/csv"
},
"routes": {
"type": "emit_table_lines",
"projection": "routes",
"mime_type": "text/csv"
}
},
"out_connections": {
"stops": {
"type": "local_file",
"encoders": [
"stops"
],
"filename": "/tmp/stops.txt"
},
"stop_times": {
"type": "local_file",
"encoders": [
"stop_times"
],
"filename": "/tmp/stop_times.txt"
},
"trips": {
"type": "local_file",
"encoders": [
"trips"
],
"filename": "/tmp/trips.txt"
},
"routes": {
"type": "local_file",
"encoders": [
"routes"
],
"filename": "/tmp/routes.txt"
}
},
"enrichments": {
"build_stop_times": {
"type": "enrichment.core.build_stop_times",
"signature": "trip_path_member_stop"
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment