참조 기반 아티팩트 추적

import wandb
run = wandb.init(project="songs", job_type="upload")
# path to my remote data directory in Google Cloud Storage
bucket = "gs://wandb-artifact-refs-public-test/whalesong"
# create a regular artifact
dataset_at = wandb.Artifact('sample_songs',type="raw_data")
# creates a checksum for each file and adds a reference to the bucket
# instead of uploading all of the contents
dataset_at.add_reference(bucket)
run.log_artifact(dataset_at)

import wandb
run = wandb.init(project="songs", job_type="show_samples")
dataset_at = run.use_artifact("sample_songs:latest")
songs_dir = dataset_at.download()
# all files available locally in songs_dir

import os
import wandb
from google.cloud import storage

run = wandb.init(project="songs", job_type="log_synth")
# full path to the specific folder of synthetic songs
# (note the "gs://" prefix for Google Storage)
synth_songs_bucket = "gs://wandb-artifact-refs-public-test/whalesong/synth"
# root of the remote bucket (note, no "gs://" prefix)
bucket_root = "wandb-artifact-refs-public-test"
dataset_at = wandb.Artifact('synth_songs',type="generated_data")

# track all the files in the specific folder of synthetic songs
dataset_at.add_reference(synth_songs_bucket)

# iterate over locations in GCP from the root of the bucket
bucket_iter = storage.Client().get_bucket(bucket_root)
song_data = []
# focus on the synth songs folder
for synth_song in bucket_iter.list_blobs(prefix="whalesong/synth"):
  # filter out any non-audio files
  if not synth_song.name.endswith(".wav"):
    continue
  
  # add a reference path for each song
  # song filenames have the form [string id]_[instrument].wav
  song_name = synth_song.name.split("/")[-1]
  song_path = os.path.join(synth_songs_bucket, song_name)
  # create a wandb.Audio object to show the audio file
  audio = wandb.Audio(song_path, sample_rate=32)
  # extract instrument from the filename
  orig_song_id, instrument = song_name.split("_")
  song_data.append([orig_song_id, song_name, audio, instrument.split(".")[0]])

# create a table to hold audio samples and metadata in columns
table = wandb.Table(data=song_data,
                    columns=["song_id", "song_name", "audio", "instrument"])
# log the table via a new artifact
songs_at = wandb.Artifact("synth_samples", type="synth_ddsp")
songs_at.add(table, "synth_song_samples")
run.log_artifact(songs_at)