W&B Artifacts Overview

# 1. Log a dataset version as an artifact
import wandb
import os

# Initialize a new W&B run to track this job
run = wandb.init(project="artifacts-quickstart", job_type="dataset-creation")

# Create a sample dataset to log as an artifact
f = open('my-dataset.txt', 'w')
f.write('Imagine this is a big dataset.')
f.close()

# Create a new artifact, which is a sample dataset
dataset = wandb.Artifact('my-dataset', type='dataset')
# Add files to the artifact, in this case a simple text file
dataset.add_file('my-dataset.txt')
# Log the artifact to save it as an output of this run
run.log_artifact(dataset)

wandb.finish()

import wandb

run = wandb.init()

artifact = wandb.Artifact("mnist", type="dataset")
artifact.add_reference("s3://moe-wandb/datasets", checksum=True)

# Track the artifact and mark it as an input to
# this run in one swoop. A new artifact version
# is only logged if the files in the bucket changed.
run.use_artifact(artifact)

# Perform training here...

artifact.add_reference("s3://moe-wandb/datasets", checksum=True, max_objects=10500000)

import wandb
run = wandb.init()
# Indicate we are using a dependency
artifact = run.use_artifact('entity/project/artifact:alias', type='artifact-type')
artifact_dir = artifact.download()

import wandb

api = wandb.Api()
artifact = api.artifact("entity/project/artifact:alias")
artifact.download()

import wandb
api = wandb.Api()
artifact = api.artifact('dummy-team/that_was_easy/s3_file_references:v0')
artifact.download()

import wandb

api = wandb.Api()
artifact = api.artifact("entity/project/artifact:alias")
# Loop through file URLs and apply filtering logic
for f in artifact.files():
	#Implement filtering logic here and custom logic for downloading or handling
	print(f.url)

import wandb
import os
import subprocess

api = wandb.Api()
art = api.artifact('dummy-team/that_was_easy/s3_file_references:v1', type='reference-dataset')
manifest_data = art.manifest.to_manifest_json()

def download_s3_contents(data):
    base_download_path = 'datasets' # Local directory to store the downloaded files

    # Extract 'ref' paths
    ref_paths = [details["ref"] for details in data["contents"].values()]

    for ref in ref_paths:
        # The ref paths in AWS S3 artifacts are usually of the form s3://bucket-name/path/to/file
        # Ensure that your ref paths are correctly formatted for AWS S3
	# Download the directory recursively using AWS CLI
        command = f"aws s3 cp --recursive {ref} {base_download_path}"
        subprocess.run(command, shell=True, check=True)
        
download_s3_contents(manifest_data)

import wandb
from datetime import timedelta

run = wandb.init(project="project", entity="entity")
artifact = wandb.Artifact(name="artifact-name", type="artifact-type")
artifact.add_file("my-file")

artifact.ttl = timedelta(days=30)  # Set TTL policy
run.log_artifact(artifact)

f"{org_entity}/wandb-registry-{registry_name}/{collection_name}"

import wandb

# Initialize a run
run = wandb.init(entity="<team_entity>", project="<project>")

# Create an artifact object
artifact = wandb.Artifact(name="<artifact_name>", type="<artifact_type>")

# Define required registry definitions 
org_entity = "<organization_entity>"
registry_name = "<registry_name>"
collection_name = "<collection_name>"
target_path = f"{org_entity}/wandb-registry-{registry_name}/{collection_name}"

# Link the artifact to a collection
run.link_artifact(artifact = artifact, target_path = target_path)

run.finish()

{ORG_ENTITY_NAME}/wandb-registry-{REGISTRY_NAME}/{COLLECTION_NAME}

import wandb
#Define team and org
TEAM_ENTITY_NAME = "<team_entity_name>"
ORG_ENTITY_NAME = "<org_entity_name>"

REGISTRY_NAME = "<registry_name>"  
COLLECTION_NAME = "<collection_name>"

run = wandb.init(
        entity=TEAM_ENTITY_NAME, project="<project_name>")

artifact = wandb.Artifact(name="<artifact_name>", type="<collection_type>")
artifact.add_file(local_path="<local_path_to_artifact>")

target_path=f"{ORG_ENTITY_NAME}/wandb-registry-{REGISTRY_NAME}/{COLLECTION_NAME}"
run.link_artifact(artifact = artifact, target_path = target_path)

import wandb

ORG_ENTITY_NAME = '<org-entity-name>'
REGISTRY_NAME = '<registry-name>'
COLLECTION_NAME = '<collection-name>'
ALIAS = '<artifact-alias>'
INDEX = '<artifact-index>'

run = wandb.init()  # Optionally use the entity, project arguments to specify where the run should be created

registered_artifact_name = f"{ORG_ENTITY_NAME}/wandb-registry-{REGISTRY_NAME}/{COLLECTION_NAME}:{ALIAS}"
registered_artifact = run.use_artifact(artifact_or_name=name)  # marks this artifact as an input to your run
artifact_dir = registered_artifact.download()  

# Artifact name with version index specified
f"{ORG_ENTITY}/wandb-registry-{REGISTRY_NAME}/{COLLECTION_NAME}:v{INDEX}"

# Artifact name with alias specified
f"{ORG_ENTITY}/wandb-registry-{REGISTRY_NAME}/{COLLECTION_NAME}:{ALIAS}"