Tabular Data Versioning and Deduplication with Weights & Biases

def make_split_artifact(run, raw_data_table, train_rows, val_rows, test_rows):
    """
    Creates a w&b artifact that contains a singular reference table (aka a ForeignIndex table).
    The ForeignIndex table has a single column that we are naming 'source'.
    It contains references to the original table (raw_data_table) for each of the splits.
    Arguments:
        run (wandb run) returned from wandb.init()
        raw_data_table (wandb Table) that contains your original tabular data
        train_rows (list of ints) indices that reference the training rows in the raw_data_table
        val_rows (list of ints) indices that reference the validation rows in the raw_data_table
        test_rows (list of ints) indices that reference the test rows in the raw_data_table
    """
    split_artifact = wandb.Artifact(
        'data-splits', type='dataset',
        description='Train, validation, test dataset splits')
    # Our data split artifact will only store index references to the original dataset table to save space
    data_table_pointer = raw_data_table.get_index() # ForeignIndex automatically references the source table
    split_artifact.add(wandb.Table(
        columns=['source'],
        data=[[data_table_pointer[i]] for i in train_rows]), 'train-data')
    split_artifact.add(wandb.Table(
        columns=['source'],
        data=[[data_table_pointer[i]] for i in val_rows]), 'val-data')
    split_artifact.add(wandb.Table(
        columns=['source'],
        data=[[data_table_pointer[i]] for i in test_rows]), 'test-data')
    run.log_artifact(split_artifact)

with wandb.init(project=PROJECT_NAME, job_type='define-transforms', config=config) as run:
    # Define an initial set of transforms that we think will be useful
    transform_dict = OrderedDict()
    transform_dict['NoneToVal'] = {
        'value': 0 # for the first pass we will replace missing values with 0
    }
    transform_dict['ToTensor'] = {
        'device': DEVICE
    }
    transform_dict['OneHot'] = {
            'num_classes': config['num_classes']
    }
    # Include an operational index to verify the order
    for key_idx, key in enumerate(transform_dict.keys()):
        transform_dict[key]['order'] = key_idx
    # Create an artifact for logging the transforms
    data_transform_artifact = wandb.Artifact(
        'data-transforms', type='parameters',
        description='Data preprocessing functions and parameters.',
        metadata=transform_dict) # Optional for viewing on the web app; the data is also stored in the txt file below
    # Log the transforms in JSON format
    with data_transform_artifact.new_file('transforms.txt') as f:
        f.write(json.dumps(transform_dict, indent=4))
    run.log_artifact(data_transform_artifact)
config.update(transform_dict) # Log the transforms in the config so that we can sweep over them in future iterations

# Now we can make the data loaders with the preprocessing pipeline
train_loader, val_loader, test_loader = make_loaders(config)