OatmealLick

...#..................#...................#......................#..............................#.............................#...
....#.......................#........#.#..............##...........#.....#..........#..........................#..................
.............#........................................................................#.........#.#..#..#..........#..............
..........#.......#..........#..................#........##...................#..............#....................................
....................#..........................................................#..#.....................................#.........
....#............................#..............................................#.....................#..........##....#..........
.......#.............#......................................................................#.............#.........#.............
....###......................#....#...........#.....#................................#.....................

OatmealLick

message = openai.ChatCompletion.create(
    model="gpt-3.5-turbo",
    messages=[
        {
            "role": "system",
            "content": "You are a code reviewer on a Merge Request on Gitlab. Your responsibility is to review "
                       "the provided code and offer"
                       "recommendations for enhancement. Identify any problematic code snippets, "
                       "highlight potential issues, and evaluate the overall quality of the code you review. "
                       "You will be given input in the format PATH: <path of the file changed>; DIFF: <diff>. "

OatmealLick

from typing import List, Any

import gitlab
import os
from itertools import dropwhile
import openai
from dataclasses import dataclass
import logging
logging.basicConfig(encoding='utf-8', level=logging.INFO)

OatmealLick

def _build_data_contract_suites(self, mappings):
    data_contract_expectation_suites = []
    for data_product, tables in mappings.items():
        for table in tables:
            suite = ExpectationSuite(
                expectation_suite_name=f"{data_product}.data_contract_schema.{table}",
                meta={
                    "great_expectations_version": "0.16.11",
                    "critical": True
                },

OatmealLick

mappings_table_id = f"astrafy-gke.data_product_mappings_{self.env}.data_product_to_public_models"
query_job = self.bigquery_client.query(f"SELECT data_product, public_models FROM {mappings_table_id}")
mappings = {r["data_product"]: r["public_models"] for r in query_job}

OatmealLick

def validate_table(data_product: str, table_id: str, env: str) -> (bool, List[str]):
    """
    data_product: str - Needed to know where to look for the table (which directory in contracts bucket)
    table_id: str - BigQuery fully qualified table id
    env: str - Choosing 'dev' or 'prd' contracts bucket
    """

    storage_client = storage.Client()
    bigquery_client = bigquery.Client()
    actual_schema = bigquery_client.get_table(table_id).schema

OatmealLick

def _compare_yaml_and_bq_schemas(expected_fields: dict, actual_schema: List[SchemaField]) -> (bool, List[str]):
    """
    Expected fields should be the YAML parsed dict.
    See `schema.yaml` for example.

    Actual schema should be the List of SchemaField, from google-cloud-bigquery.
    """
    actual_fields = {field.name: field for field in actual_schema}
    all_messages = []
    aggregated_result = True

OatmealLick

fields:
  - name: _string
    type: string
  - name: _bytes
    type: bytes
  - name: _integer
    type: integer
  - name: _int64
    type: integer
  - name: _float64