Google Cloud

All the goodness of Google Cloud Platform
# add project id to the environment variable
export PROJECT_ID=$(gcloud config list --format 'value(core.project)')

Build docker images and push to container registry

# authentication for docker
# from
gcloud auth configure-docker
# build a docker image locally and push to GCP container registry
docker build -t .
gcloud docker -- push
# or just use cloud builds which will automatically store the image in registry
gcloud builds submit --timeout 2h --tag .
# pull image for usage
gcloud docker -- pull
# -- (double dashes) are used to pass parameters to docker.
# If advanced authentication is set up properly, you can skip using
# gcloud docker
# and just use
# docker


Copy data from VM to Local
Second Tab
Copy data from VM to Local
# copy files from a VM to local
gcloud compute scp --recurse my-vm://home/vikramtheone1/my-files/ ./ --zone=us-central1-b
Second Tab


De-dupe complete table
Second Tab
De-dupe complete table
# this will de-deup data based on all rows matching each other
CREATE OR REPLACE TABLE `project.dataset.table`
PARTITION BY timestamp
AS SELECT DISTINCT * FROM `project.dataset.table`
Second Tab