From 6fda123bce74078734224ea677ed112ab70ec5f0 Mon Sep 17 00:00:00 2001 From: Julian Bright Date: Wed, 2 Jun 2021 17:53:58 +1000 Subject: [PATCH] Adding conda efs env --- README.md | 4 +- examples/conda-efs-image/Dockerfile | 52 ++++++++++ examples/conda-efs-image/README.md | 95 ++++++++++++++++++ .../app-image-config-input.json | 16 +++ .../conda-efs-image/create-domain-input.json | 19 ++++ .../custom_kernel_spec/kernel.json | 9 ++ .../custom_kernel_spec/logo-32x32.png | Bin 0 -> 1084 bytes .../custom_kernel_spec/logo-64x64.png | Bin 0 -> 2180 bytes .../conda-efs-image/update-domain-input.json | 12 +++ 9 files changed, 206 insertions(+), 1 deletion(-) create mode 100644 examples/conda-efs-image/Dockerfile create mode 100644 examples/conda-efs-image/README.md create mode 100644 examples/conda-efs-image/app-image-config-input.json create mode 100644 examples/conda-efs-image/create-domain-input.json create mode 100644 examples/conda-efs-image/custom_kernel_spec/kernel.json create mode 100644 examples/conda-efs-image/custom_kernel_spec/logo-32x32.png create mode 100644 examples/conda-efs-image/custom_kernel_spec/logo-64x64.png create mode 100644 examples/conda-efs-image/update-domain-input.json diff --git a/README.md b/README.md index 20449e0..876216f 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,9 @@ This repository contains examples of Docker images that are valid custom images - [r-image](examples/r-image) - This example contains the `ir` kernel and a selection of R packages, along with the AWS Python SDK (boto3) and the SageMaker Python SDK which can be used from R using `reticulate` - [rapids-image](examples/rapids-image) - This example uses the offical rapids.ai image from Dockerhub. Use with a GPU instance on Studio - [scala-image](examples/scala-image) - This example adds a Scala kernel based on [Almond Scala Kernel](https://almond.sh/). -- [tf2.3-image](examples/tf23-image) - This examples uses the official TensorFlow 2.3 image from DockerHub and demonstrates bundling custom files along with the image. +- [tf2.3-image](examples/tf23-image) - This example uses the official TensorFlow 2.3 image from DockerHub and demonstrates bundling custom files along with the image. +- [conda-efs-image](examples/conda-efs-image) - This example creates a custom image in Amazon SageMaker Studio which will load a `custom` conda environment from a users [Elastic File System](https://docs.aws.amazon.com/sagemaker/latest/dg/studio-tasks-manage-storage.html) (EFS) home folder. + #### One-time setup All examples have a one-time setup to create an ECR repository diff --git a/examples/conda-efs-image/Dockerfile b/examples/conda-efs-image/Dockerfile new file mode 100644 index 0000000..4abb0d5 --- /dev/null +++ b/examples/conda-efs-image/Dockerfile @@ -0,0 +1,52 @@ +FROM public.ecr.aws/ubuntu/ubuntu:20.04 + +ARG NB_USER="sagemaker-user" +ARG NB_UID="1000" +ARG NB_GID="100" +ARG NB_ENV="custom" + +###################### +# OVERVIEW +# 1. Creates the `sagemaker-user` user with UID/GID 1000/100. +# 2. Ensures this user can `sudo` by default. +# 5. Make the default shell `bash`. This enhances the experience inside a Jupyter terminal as otherwise Jupyter defaults to `sh` +###################### + +# Setup the "sagemaker-user" user with root privileges. +RUN \ + apt-get update && \ + apt-get install -y sudo wget nano && \ + useradd -m -s /bin/bash -N -u $NB_UID $NB_USER && \ + chmod g+w /etc/passwd && \ + echo "${NB_USER} ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers && \ + # Prevent apt-get cache from being persisted to this layer. + rm -rf /var/lib/apt/lists/* + +# installing miniconda +RUN wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh +RUN bash Miniconda3-latest-Linux-x86_64.sh -b -p /miniconda +ENV PATH=$PATH:/miniconda/condabin:/miniconda/bin + +# install libraries for the base environment +RUN conda install ipykernel + +# update kernelspec to load our custom environment strored on EFS +RUN rm -rf /miniconda/share/jupyter/kernels/python3 +# write new kernel which initialises uses conda run for custom env +# see: https://github.com/ipython/ipykernel/issues/416 +COPY custom_kernel_spec/ /miniconda/share/jupyter/kernels/python3 + +# Make the default shell bash (vs "sh") for a better Jupyter terminal UX +ENV SHELL=/bin/bash \ + NB_USER=$NB_USER \ + NB_ENV=$NB_ENV \ + NB_UID=$NB_UID \ + NB_GID=$NB_GID \ + HOME=/home/$NB_USER + +# Set the conda envs path to map to EFS (without of requiring .condarc) +ENV CONDA_AUTO_ACTIVATE_BASE=false \ + CONDA_ENVS_PATH=/home/$NB_USER/.conda/envs + +WORKDIR $HOME +USER $NB_UID \ No newline at end of file diff --git a/examples/conda-efs-image/README.md b/examples/conda-efs-image/README.md new file mode 100644 index 0000000..227b9c0 --- /dev/null +++ b/examples/conda-efs-image/README.md @@ -0,0 +1,95 @@ +## Conda EFS Image + +### Overview + +This example creates a custom image in Amazon SageMaker Studio which will load a `custom` conda environment from a users [Elastic File System](https://docs.aws.amazon.com/sagemaker/latest/dg/studio-tasks-manage-storage.html) (EFS) home folder. + +The Dockerfile is built from `ubuntu:20.04` and installs the latest [Minconda3](https://docs.conda.io/en/latest/miniconda.html) distribution. + +### Building the image + +Build the Docker image and push to Amazon ECR. +``` +# Modify these as required. The Docker registry endpoint can be tuned based on your current region from https://docs.aws.amazon.com/general/latest/gr/ecr.html#ecr-docker-endpoints +REGION= +ACCOUNT_ID= + +# Build the image +IMAGE_NAME=custom-conda-efs +aws --region ${REGION} ecr get-login-password | docker login --username AWS --password-stdin ${ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/smstudio-custom +docker build . -t ${IMAGE_NAME} -t ${ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/smstudio-custom:${IMAGE_NAME} --build-arg NB_ENV=custom +``` + +NOTE: The `NB_ENV` maps to the conda environment which will need to be created on EFS (see below) + +``` +docker push ${ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/smstudio-custom:${IMAGE_NAME} +``` + +### Using it with SageMaker Studio + +Create a SageMaker Image with the image in ECR. + +``` +# Role in your account to be used for the SageMaker Image +ROLE_ARN= + +aws --region ${REGION} sagemaker create-image \ + --image-name ${IMAGE_NAME} \ + --display-name "Conda EFS env" \ + --role-arn ${ROLE_ARN} + +aws --region ${REGION} sagemaker create-image-version \ + --image-name ${IMAGE_NAME} \ + --base-image "${ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/smstudio-custom:${IMAGE_NAME}" + +# Verify the image-version is created successfully. Do NOT proceed if image-version is in CREATE_FAILED state or in any other state apart from CREATED. +aws --region ${REGION} sagemaker describe-image-version --image-name ${IMAGE_NAME} +``` + +Create an AppImageConfig for this image. + +``` +aws --region ${REGION} sagemaker create-app-image-config --cli-input-json file://app-image-config-input.json +``` + +Create a Domain, providing the SageMaker Image and AppImageConfig in the Domain creation. Replace the placeholders for VPC ID, Subnet IDs, and Execution Role in `create-domain-input.json`. + +``` +aws --region ${REGION} sagemaker create-domain --cli-input-json file://create-domain-input.json +``` + +If you have an existing Domain, you can use the `update-domain` command. + +``` +DOMAIN_ID= +aws --region ${REGION} sagemaker update-domain --domain-id ${DOMAIN_ID} --cli-input-json file://update-domain-input.json +``` + +### Create your custom conda environment on EFS + +Use the [Amazon SageMaker Studio Launcher](https://docs.aws.amazon.com/sagemaker/latest/dg/studio-launcher.html) to open a **System terminal** and run the following commands to setup your custom environment: + +```bash +mkdir -p ~/.conda/envs +conda create -p ~/.conda/envs/custom +conda activate custom # or ". activate ~/.conda/envs/custom" +conda install ipykernel +``` + +### Verifying your installation + +Use the [Amazon SageMaker Studio Launcher](https://docs.aws.amazon.com/sagemaker/latest/dg/studio-launcher.html) to launch a **Notebook** for your *Conda EFS env* SageMaker Image and run the following command in the first cell to verify you are now using a `custom` conda environment: + +``` +!conda info +``` + +### Installing additional libraries + +Use the [Amazon SageMaker Studio Launcher](https://docs.aws.amazon.com/sagemaker/latest/dg/studio-launcher.html) to launch an **Image terminal** and run the following commands to install new libraries in your custom environment such as `numpy`: + +```bash +. activate /home/$NB_USER/.conda/envs/$NB_ENV +conda install numpy +``` \ No newline at end of file diff --git a/examples/conda-efs-image/app-image-config-input.json b/examples/conda-efs-image/app-image-config-input.json new file mode 100644 index 0000000..5a08191 --- /dev/null +++ b/examples/conda-efs-image/app-image-config-input.json @@ -0,0 +1,16 @@ +{ + "AppImageConfigName": "custom-conda-efs-env-config", + "KernelGatewayImageConfig": { + "KernelSpecs": [ + { + "Name": "python3", + "DisplayName": "Python 3 (custom EFS env)" + } + ], + "FileSystemConfig": { + "MountPath": "/home/sagemaker-user", + "DefaultUid": 1000, + "DefaultGid": 100 + } + } +} diff --git a/examples/conda-efs-image/create-domain-input.json b/examples/conda-efs-image/create-domain-input.json new file mode 100644 index 0000000..2498c1d --- /dev/null +++ b/examples/conda-efs-image/create-domain-input.json @@ -0,0 +1,19 @@ +{ + "DomainName": "domain-with-conda-efs-image", + "VpcId": "", + "SubnetIds": [ + "" + ], + "DefaultUserSettings": { + "ExecutionRole": "", + "KernelGatewayAppSettings": { + "CustomImages": [ + { + "ImageName": "custom-conda-efs", + "AppImageConfigName": "custom-conda-efs-env-config" + } + ] + } + }, + "AuthMode": "IAM" +} \ No newline at end of file diff --git a/examples/conda-efs-image/custom_kernel_spec/kernel.json b/examples/conda-efs-image/custom_kernel_spec/kernel.json new file mode 100644 index 0000000..40ed268 --- /dev/null +++ b/examples/conda-efs-image/custom_kernel_spec/kernel.json @@ -0,0 +1,9 @@ +{ + "argv": [ + "bash", + "-c", + "conda run -n $NB_ENV python -m ipykernel_launcher -f '{connection_file}'" + ], + "display_name": "Conda EFS env", + "language": "python" +} \ No newline at end of file diff --git a/examples/conda-efs-image/custom_kernel_spec/logo-32x32.png b/examples/conda-efs-image/custom_kernel_spec/logo-32x32.png new file mode 100644 index 0000000000000000000000000000000000000000..be81330765764699553aa4fbaf0e9fc27c20c6d2 GIT binary patch literal 1084 zcmV-C1jGA@P)enw2jbMszQuf3kC$K7$S;4l;TgSRfzha5>pgWAEY9PR!IdB zTSZXtp`b02h)|SJ3#AW|AKF?KgNSQ|Sg=ZCgHaT%F`4#g>iG8;N__GBLh26(2qOGO9};SPeUDLyV^m!K($s69;fB|`Ui z{nqhFk+};I5Vb+1*IC+gaNEtF()dX{`(!1eUb?=>+~p#JOj-qUi2^^^uzi1p(thMz&#&LJq>Cf)~tBhxq*;Npy$=mheX>2t4(OR zWk&s74VR$m@6rlD?Nud*cEGO2$>|mV&tzP1%j+W-N_;a>$_%)&Yn?|hX(50fV5s); zkLsKLb20?nJo-eIQ&vLU?~T?v{=JUtFa!EFC;;*i2@lY(#8Ur2b{` z!nc_6C42;g?mDnyRp9)U84ZxUv=Ja10XDYX;KZ|EPJ`h_&;S{#m9Q!a*xC#MiI?P; zx4sNs;+Uif!Da~pAQU}S)ww^M;qb(^FD`~`s1D2+foklsECF&ZZKas%kF~bU-M9bY zuhs+V2CzISGy`A&Lkq;MkgWkjD)R)1WqC_*Tx45LdH=lV+}XPaAFS+wus(ZG#IuZp zEE@YdBSMkKnX~3J?j7u_^kl&mQ+7t_i^t4YG6X0cS+J89bl~_Igc~wh(?=P_08}Iv z0NHqkz|x<~Z;3paR=+czhC^#TYlWDdd@Rc|#cCUooxt4edl>=;-neznjL)SlXtdOh z=2NAO%Gxj%BLM->i|(q=eePLs=%wD>*F6312}yTRxn%!IzZtmkN`YjQBMNkckc4h;pSXO%%?N2y_ccz zS`INlItXC6DR;umS}Mn43NzsR7MS0Sf|rrv1n7UvdO9UC3&XB+{A~zNMyyXY@lF_q zps;z-9S*u(m1{=;T?YYxd%vmwj5N7<3lv^}?EK6DlWbFPZoBI|w5zEE06;(VF2nD? z_QUyZi0eRG2jDb-NyvSR5{_bd`5o6W`WOCh1>4`s79R;zVm_k)0000kjcw83I)rwURf9H)0d)l3>^8*`$3&wplXaSnv^ouL zxig617>J8x{$<2zvZ44vm&sPJz*Z;|)^sj29S|e(QD`@&rR&E%&(A;Zx#ym9?>Xnb z=k|6x#=dRS_rB-ex99mi&+qvXHKxY@^N`8h{N|r@TsA(& zsCpk!BK%oN(i-QUbD69cd?H!sn{mG-Lrs4l70Gd-TRSnnlw<)m#)CQ1364@U( zb1huc+%2C?f zYjwl_PTT;XJ$4oVU=Be51c+U`UEX_ls%aSHu0jnXMCH=*+Sd}C2irp2UqB=Z0E)N85&+GM z>q^`|nwHj#MQ}!_hFxHI0P?d05b<<^{$@L)xRXP$*7NMe_Al`SAe_UPXbALJOH3_5 zcM?1d0-}ThP+N;&R(k{$P!RUyBLuGx7u*NjI0EqWx*LBO^)ny+&f^)CC}~0x8ViOeXmOp`hB@Wk%DqXy3C1Q0?$fKnaUFPm1OP-ZjVK`deF} zSeAF2mylo&RQ`&~-?2v|r4t6AY0JJPRN1JijUXW&kBk6^2Cvr^I{u5UuqP$>16T2K z9R$k@xromL3Y>lI8J_*t?K0<)3neE)OPIZA`y$|W32O|S;>(;-_BoaG7O_=2G z6D)9yzzx@Wf#9y!>3jH(JLX0Lz*6}#sWZF@h^aPF)_fq;^c^8JPiTh*0JRcGe<2b8 zN_@jF0rBt^lR=9@fPBV9TT3%D0)}bdo{O3TaO38^?3k0H{bUT-qpE!%+$xpS2LPf1an-UJ2DJ9KqouI6R;TMiW;X0gzCw zHO|Y+R^XVXy4>IM=$idVj4jUz?GhXz)&RZ6C=nuAOFRF5GYcGpaQ8++^bVf8D~Ysh zasY5*fBszU=;2(eHKTx{cJgCCqK3OyNG?6L{qEzi@F-xtJB056lt^D=Mgd{1M;|3o zptQ9-Tf6}9DG0x>)iWA;*7d!}f34XL)z1YaJw+(tZvmBs7Qne4&B4c^71J}j0Cl!mHAtQyc|{3a zzhEhE=-#}lmuK6SVomEdD6U096Gc<`?9IYNt09igBXq$&uNwIPk|#@Za%kz^ysDSy z+SWt37r+OM+U|uhJI|3tadcq`kq(&o0OEv1c4+!|*N<=iE&E$ngIs6G>;UsEYRUoH z*N{CGAkP{BAQ=ioDsa;2iU)Z9+n0m7&G0!|IACWkdlBI1w@S4<6a_#XeAP z1@TTJt)oc(Zd&9NrG)FXraO%+ph_!V8AqA`#S;PpD4=AwE!!e+(HZRH`J4Q`%$PKn zL#RLx{&wZdvT~>OrXG{ynQ!)hTxeLDW{is=avgT_Q@X{_ryQSRf-z;cCzzZ%57>p+XNOwhgQWFSDdeo<;8g((CJEj(Z4)c6IEc3%k9{YIG zk+*m8hahOo-7ycwG7kU%o^1X(sCP!|<+23tKd4KhH8=|#dkr8hdCPys`Kq?qW`a42rV{8owiaTo2X%UpUcJedmjJmB_0Mh> zDfdCyN&K%dp1k=ojE<}Z_*K9@aFMV5@X-t5FOkM$vasuX>}!EgFkb%DENHq8U>%?f zGQUv=A_?Fk1g}BS5Ab;i4xv&G$^7TeU}{W_sWCMsdHfgT%>1XE)oy