Skip to content
Snippets Groups Projects
Commit 294aa8d7 authored by Jedrzej Rybicki's avatar Jedrzej Rybicki
Browse files

simple dislib example

parent 714c6479
Branches
Tags
No related merge requests found
FROM ubuntu:latest
RUN apt update && apt-get install -y git wget unzip openjdk-8-jdk graphviz xdg-utils libtool automake build-essential python3 python3-dev libboost-serialization-dev libboost-iostreams-dev libxml2 libxml2-dev csh gfortran libgmp3-dev flex bison texinfo python3-pip libpapi-dev && rm -rf /var/lib/apt/lists/*
RUN wget https://services.gradle.org/distributions/gradle-5.4.1-bin.zip -O /opt/gradle-5.4.1-bin.zip && unzip /opt/gradle-5.4.1-bin.zip -d /opt
ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
ENV GRADLE_HOME=/opt/gradle-5.4.1
ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/python3.10/dist-packages/pycompss/COMPSs/Bindings/bindings-common/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/
RUN pip install pycompss dislib jupyter mlflow==1.30.0
EXPOSE 8888
CMD jupyter notebook --allow-root --no-browser --port=8888 --ip=0.0.0.0 --notebook-dir=/notebooks/
%% Cell type:markdown id:4061677f tags:
## Example of dislib model upload to mlflow
It uses docker to run dislib (image can be found in dislibimage), build it with
```
docker build . -t mydislib
```
and start with:
```
docker run -p 8888:8888 -v `pwd`:/notebooks/ mydislib
```
%% Cell type:code id:24716b0f tags:
``` python
import numpy as np
from sklearn.linear_model import LogisticRegression
import mlflow
import mlflow.sklearn
import matplotlib.pyplot as plt
import warnings
warnings.filterwarnings('ignore')
import pycompss.interactive as ipycompss
```
%% Cell type:code id:7eba2399 tags:
``` python
import mlflow
import mlflow.pyfunc
class PyCompssWrapper(mlflow.pyfunc.PythonModel):
def __init__(self, md):
super().__init__()
self.model = md
def predict(self, context, model_input):
return self.model.predict(model_input)
@staticmethod
def get_pip():
return [f"pycomps", f"dislib=={ds.__version__}"]
```
%% Cell type:code id:7cdf795d tags:
``` python
#ipycompss.start(graph=True, monitor=1000)
ipycompss.start(graph=True,
project_xml='../xml/project.xml',
resources_xml='../xml/resources.xml')
```
%% Output
********************************************************
**************** PyCOMPSs Interactive ******************
********************************************************
* .-~~-.--. ______ ___ *
* : ) |____ \ / | *
* .~ ~ -.\ /.- ~~ . __) | /_/| | *
* > `. .' < |__ | | | *
* ( .- -. ) ____) | _ | | *
* `- -.-~ `- -' ~-.- -' |______/ |_| |_| *
* ( : ) _ _ .-: *
* ~--. : .--~ .-~ .-~ } *
* ~-.-^-.-~ \_ .~ .-~ .~ *
* \ \ ' \ '_ _ -~ *
* \`.\`. // *
* . - ~ ~-.__\`.\`-.// *
* .-~ . - ~ }~ ~ ~-.~-. *
* .' .-~ .-~ :/~-.~-./: *
* /_~_ _ . - ~ ~-.~-._ *
* ~-.< *
********************************************************
* - Starting COMPSs runtime... *
* - Log path : /root/.COMPSs/Interactive_03/
* - PyCOMPSs Runtime started... Have fun! *
********************************************************
%% Cell type:code id:9ccb1851 tags:
``` python
from dislib.cluster import KMeans
import dislib as ds
from sklearn.datasets import make_blobs
import matplotlib.pyplot as plt
```
%% Cell type:code id:e9a55fb8 tags:
``` python
mlflow.set_tracking_uri('https://zam10017.zam.kfa-juelich.de/')
mlflow.set_experiment('dislib-tests-clean')
```
%% Output
<Experiment: artifact_location='mlflow-artifacts:/5', creation_time=1680077947468, experiment_id='5', last_update_time=1680077947468, lifecycle_stage='active', name='dislib-tests-clean', tags={}>
%% Cell type:code id:e749733a tags:
``` python
x, y = make_blobs(n_samples=1500)
x_ds = ds.array(x, block_size=(500, 2))
```
%% Cell type:code id:297dc16c tags:
``` python
mlflow.end_run()
```
%% Cell type:code id:3f2b341d tags:
``` python
mlflow.start_run()
```
%% Output
<ActiveRun: >
%% Cell type:code id:213adb03 tags:
``` python
km = KMeans(n_clusters=3)
mlflow.log_param('k', value=3)
km.fit(x_ds)
y_pred = km.predict(x_ds)
centers = km.centers
#log model
wrapped = PyCompssWrapper(km)
model_info = mlflow.pyfunc.log_model(artifact_path='dislibmodel',
python_model=wrapped,
pip_requirements=PyCompssWrapper.get_pip() )
```
%% Output
INFO: The ERRMGR displayed some error or warnings.
%% Cell type:code id:642782c7 tags:
``` python
fig, axs = plt.subplots(1)
axs.scatter(x[:, 0], x[:, 1], c=y_pred.collect())
axs.scatter(centers[:, 0], centers[:, 1], c='red')
#log figure
mlflow.log_figure(fig, 'centers.png')
```
%% Output
%% Cell type:code id:d1025134 tags:
``` python
# log compss config
mlflow.log_artifacts('../xml/')
```
%% Cell type:code id:97282393 tags:
``` python
ipycompss.stop()
```
%% Output
********************************************************
*************** STOPPING PyCOMPSs ******************
********************************************************
Checking if any issue happened.
Warning: some of the variables used with PyCOMPSs may
have not been brought to the master.
********************************************************
%% Cell type:code id:7fcdfcdd tags:
``` python
mlflow.end_run()
```
%% Cell type:code id:9ac26e93 tags:
``` python
```
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<Project>
<MasterNode>
<Processor Name="MainProcessor">
<ComputingUnits>4</ComputingUnits>
<Architecture>amd64</Architecture>
<Speed>3.0</Speed>
</Processor>
<Memory>
<Size>16</Size>
</Memory>
<Storage>
<Size>200.0</Size>
</Storage>
</MasterNode>
</Project>
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<ResourcesList>
</ResourcesList>
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment