updates to docker image

jhanna-kx · jhanna-kx · commit 6428797ad091 · 2022-11-11T16:04:58.000Z
osx-arm64 packages revert broken embedpy under python commit compiler warnings remove tensorflow tests restrict to python 3.9 and 3.10 retry numpy import on osx due to warnings generated in more recent versions of numpy RuntimeError: Polyfit sanity test emitted a warning, most likely due to using a buggy Accelerate backend. If you compiled yourself, more information is available at: https://numpy.org/doc/stable/user/building.html#accelerated-blas-lapack-libraries Otherwise report this to the vendor that provided NumPy.
diff --git a/.travis.yml b/.travis.yml
@@ -1,5 +1,8 @@
 stage: standalone
 language: c
+dist:
+ - focal
+osx_image: xcode12.5
 os:
 - linux
 - osx
@@ -15,7 +18,7 @@ before_install:
       wget https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O miniconda.sh;
     fi
   - bash miniconda.sh -b -p $HOME/miniconda
-  - export PATH="$HOME/miniconda/bin:$PATH"
+  - source "$HOME/miniconda/etc/profile.d/conda.sh"
   - hash -r
   - conda config --set always_yes yes --set changeps1 no
   - conda update -q conda
@@ -25,40 +28,19 @@ before_script:
   - perl -p -i -e 's/EMBEDPYVERSION/`\$\"'${IMPLEMENTATION}'\"/g' p.q
 
 script:
-  - if [ $TRAVIS_OS_NAME = osx ]; then pyenv init - > pyenv.init && source pyenv.init ; fi
-  - if [ $TRAVIS_OS_NAME = linux ]; then QLIBDIR=l64;OD=$L64; elif [ $TRAVIS_OS_NAME = osx ]; then QLIBDIR=m64;OD=$M64; else echo "unknown OS ('$TRAVIS_OS_NAME')" >&2; exit 1; fi; export QLIBDIR
-  - mkdir q && export QHOME=$(pwd)/q && export PATH=$QHOME/$QLIBDIR:$PATH
-  - if [[ "x$OD" != "x" ]]; then
-      curl -u $NEXUS_USER:$NEXUS_PASS -o ${QHOME}/q.zip -L $OD;
-      unzip -d q q/q.zip;
-    else
-      echo No kdb+;
-    fi
-
+  - if [ $TRAVIS_OS_NAME = linux ]; then QLIBDIR=l64; elif [ $TRAVIS_OS_NAME = osx ]; then QLIBDIR=m64; else echo "unknown OS ('$TRAVIS_OS_NAME')" >&2; exit 1; fi; export QLIBDIR
+  - conda create -n bld -c kx/label/dev kdb ;
+  - conda activate bld
   - make p.so
-  - cp [ml]64/p.so q/$QLIBDIR;
+  - cp [ml]64/p.so $QHOME/$QLIBDIR;
   - tar czf embedPy_$TRAVIS_OS_NAME-$TRAVIS_BRANCH.tgz p.q p.k test.q tests [ml]64/p.so LICENSE README.md
   - if [[ "x$QLIC_KC" != "x" ]]; then
-      echo -n $QLIC_KC |base64 --decode > q/kc.lic;
-      pip install --upgrade pip;
-      pip install --upgrade setuptools;
-      pip -q install -r tests/requirements.txt;
-      q test.q -s 4 -q;
-    else
-      echo No kdb+, no tests;
-    fi
-
-  ## virtualenv
-  - pip install virtualenv;
-  - virtualenv travisEnv;
-  - source travisEnv/bin/activate
-  - echo $(which python)
-  - if [[ "x$QLIC_KC" != "x" ]]; then
-      echo -n $QLIC_KC |base64 --decode > q/kc.lic;
+      echo -n $QLIC_KC |base64 --decode > $QHOME/kc.lic;
+      export QLIC_KC=;
       pip install --upgrade pip;
-      pip install --upgrade setuptools;
       pip -q install -r tests/requirements.txt;
       q test.q -s 4 -q;
+      echo DONE TEST;
     else
       echo No kdb+, no tests;
     fi
@@ -115,8 +97,9 @@ jobs:
         - if [ $TRAVIS_OS_NAME = linux ]; then QLIBDIR=l64; elif [ $TRAVIS_OS_NAME = osx ]; then QLIBDIR=m64; fi; export QLIBDIR
         - if [[ "x$QLIC_KC" != "x" ]]; then mkdir -p q; echo -n $QLIC_KC |base64 --decode > q/kc.lic; export QLIC=$($PREFIX)/q; fi
         - export EMBEDPY_VERSION=$TRAVIS_BRANCH
-        - conda build conda-recipe --output -c kx > packagenames.txt
-        - conda build -c kx conda-recipe --no-long-test-prefix
+        - conda build conda-recipe --output -c kx/label/dev > packagenames.txt
+        - if [[  "$TRAVIS_OS_NAME" == "osx" ]]; then export CHANS="-c conda-forge"; fi
+        - conda build -c kx/label/dev $CHANS conda-recipe --no-long-test-prefix
       deploy:
         provider: script
         script:
diff --git a/appveyor.yml b/appveyor.yml
@@ -3,6 +3,9 @@ image:
 - Visual Studio 2017
 artifacts:
 - path: embedPy_windows-$(appveyor_repo_tag_name).zip
+init:
+  - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
+
 skip_non_tags: false
 platform: x64
 build_script:
diff --git a/conda-recipe/conda_build_config.yaml b/conda-recipe/conda_build_config.yaml
@@ -1,6 +1,8 @@
+CONDA_BUILD_SYSROOT:
+ - /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk # [osx]
+target_platform: # [osx]
+ - osx-64        # [osx]
+ - osx-arm64     # [osx]
 python:
-  - 3.5 [unix]
-  - 3.6
-  - 3.7
-  - 3.8
   - 3.9
+  - 3.10
diff --git a/conda-recipe/run_test.sh b/conda-recipe/run_test.sh
@@ -1,9 +1,15 @@
 #!/bin/bash
-if [ -e ${QLIC}/kc.lic ]
+if [[ $SUBDIR == $build_platform ]]
 then
-  q conda-recipe/prep_requirements.q -q
-  conda install -y -q --file tests/requirements_filtered.txt
-  q test.q -s 4 -q;
+ if [ -e ${QLIC}/kc.lic ]
+ then
+   q conda-recipe/prep_requirements.q -q
+   conda install -y -q --file tests/requirements_filtered.txt
+   q test.q -s 4 -q;
+ else
+   echo No kdb+, no tests;
+ fi
 else
-  echo No kdb+, no tests;
+ echo cross compile, no tests
 fi
+
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -56,34 +56,26 @@ RUN apt-get -yy --option=Dpkg::options::=--force-unsafe-io --no-install-recommen
 
 RUN passwd -d root
 RUN useradd -s /bin/bash -U -m kx
-
-ENV QHOME=/opt/kx/q
-ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}/opt/conda/lib
-
-RUN mkdir -p /opt/kx
+ENV KX_IN_DOCKER=Y
 
 COPY --from=anaconda /opt/conda /opt/conda
 
 RUN . /opt/conda/etc/profile.d/conda.sh \
 	&& conda update -n base conda \
-	&& conda clean -y --all \ 
-	&& conda install -y numpy==1.20.3 \
-	&& conda install -c kx kdb	
+	&& conda clean -y --all
 
-RUN  mv /opt/conda/q /opt/kx
-COPY docker/q.wrapper /usr/local/bin/q
-COPY docker/kc.lic.py /opt/kx/
-
-COPY --from=embedpy /opt/kx/embedPy /opt/kx/embedPy
-RUN ln -s -t $QHOME/l64 /opt/kx/embedPy/l64/p.so \
-	&& ln -s -t $QHOME /opt/kx/embedPy/p.q /opt/kx/embedPy/p.k
 
 COPY docker/profile.sh /etc/profile.d/kx.sh
+COPY --from=embedpy /opt/kx/embedPy /opt/kx/embedPy
 
 USER kx
 
 RUN . /opt/conda/etc/profile.d/conda.sh \
-	&& conda create -y -n kx python=3.9 --no-default-packages
+	&& conda create -y -n kx python --no-default-packages \
+        && conda activate kx \
+        && conda install -c kx kdb \
+        && ln -s -t $QHOME/l64 /opt/kx/embedPy/l64/p.so \
+        && ln -s -t $QHOME /opt/kx/embedPy/p.q /opt/kx/embedPy/p.k
 
 USER root
 
diff --git a/docker/init b/docker/init
@@ -3,7 +3,7 @@
 set -eu
 
 if ! test -x "$1"; then
-	SH=$(which "$1")
+	SH=$(which "$1") || SH=$1
 	shift
 	set -- $SH "$@"
 fi
@@ -19,6 +19,5 @@ HOME=/home/kx
 cd $HOME
 
 export QLIC=$HOME
-chpst -u kx /bin/sh -l -c 'exec python /opt/kx/kc.lic.py'
 
 exec chpst -u kx "$@"
diff --git a/p.q b/p.q
@@ -4,7 +4,6 @@ version:@[{EMBEDPYVERSION};0;`development];
 
 o:first string .z.o;
 $[o="w";if[3.6>.z.K;'`$"kdb+ version must be 3.6+"];if[3.5>.z.K;'`$"kdb+ version must be 3.5+"]];
-if[(o="w")&count getenv[`UNDER_PYTHON];'"embedPy running within a Python process not supported for Windows"];
 if[not .P.loaded:-1h=type@[`.p@;`numpy;`];
  sc:{"'",x,"'.join([__import__('sysconfig').get_config_var(v)for v in",ssr[.j.j y;"\"";"'"],"])"};pr:{"print(",x,");"};
  c:"-c \"",pr["'.'.join([str(getattr(__import__('sys').version_info,x))for x in ['major','minor']])"],"\"2>",$[o="w";"nul <nul";"/dev/null"];
@@ -119,10 +118,3 @@ if[not loaded;if[not count .p.import[`sys][`:argv]`;.p.import[`sys][:;`:argv;enl
 
 / VirtualEnv warning for windows users
 if[.P.env&.z.o like"w*";-2"Warning:\n\tVirtual Environments not supported for embedPy on Windows.\n\tUsing the 'BASE' version of Python, not the virtual environment Python";]
-
-/ Add warning for unsupported numpy 1.22
-{np:@[.p.import;`numpy;{(::)}];if[np~(::);:np];
- if[any 1 21<"J"$-1_vs["."]np[`:__version__]`;
- -2"Warning:\n\tDue to a bug in the numpy C api, conversions between q and numpy objects for 'numpy>=1.22'\n\tare unsupported at this time. Please downgrade your numpy version.";]
- }`
-
diff --git a/py.c b/py.c
@@ -49,17 +49,13 @@ Z K3(call){P a,k,p;I g;P(!pq(x)||!pq(z)&&z->t!=XD,ktrr())g=g1();a=pq(y)?p1(pg(y)
 Z K1(setconv){P(xt<100||xt>111,ktrr())r0(cf);cf=r1(x);R 0;}Z K1(getconv){R r1(cf);}Z K1(isp){R kb(pq(x));}
 Z K1(setpyerr){P(xt!=-KJ,ktrr())P(!errfmt,krr("traceback.format_exception"))pyerr=xj;R 0;}Z K1(getpyerr){R kj(pyerr);}
 
-ZV*t;EXP K3(init){ZI i=0;I f,g;S l,h,hh;K n,v;P a,b,pyhome;P(i,0)l=zs(x),h=zs(y),hh=zs(z);f=pyl(l);free(l);
- P(!f,krr("libpython"))
- if(!Py_IsInitialized()){
-  Py_SetPythonHome(Py_DecodeLocale(h,0));Py_SetProgramName(Py_DecodeLocale(hh,0));Py_InitializeEx(0);
-  if(PyEval_ThreadsInitialized()&&!PyGILState_Check())t0(PyGILState_GetThisThreadState());PyEval_InitThreads();
-  }
+ZV*t;EXP K3(init){ZI i=0;I f;S l,h,hh;K n,v;P a,b,c=0;P(i,0)l=zs(x),h=zs(y),hh=zs(z);f=pyl(l);free(l);P(!f,krr("libpython"))
+ Py_SetPythonHome(Py_DecodeLocale(h,0));Py_SetProgramName(Py_DecodeLocale(hh,0));free(hh);free(h);Py_InitializeEx(0);if(PyEval_ThreadsInitialized()&&!PyGILState_Check())t0(PyGILState_GetThisThreadState());PyEval_InitThreads();
  M=PyModule_GetDict(PyImport_AddModule("__main__"));cf=k(0,"::",0);n=ktn(KS,0);v=ktn(0,0);
- if(a=PyImport_ImportModule("numpy.core.multiarray")){N=PyCapsule_GetPointer(b=PyObject_GetAttrString(a,"_ARRAY_API"),0);if(!N||!pyn(N))N=0;p0(b);p0(a);}PyErr_Clear();
+ if((a=PyImport_ImportModule("numpy.core.multiarray"))||(PyErr_Print(),c=PyImport_ImportModule("numpy.core.multiarray"))){N=PyCapsule_GetPointer(b=PyObject_GetAttrString(a?a:c,"_ARRAY_API"),0);if(!N||!pyn(N))N=0;p0(b);p0(a);p0(c);}PyErr_Clear();
  if(a=PyImport_ImportModule("traceback")){errfmt=PyObject_GetAttrString(a,"format_exception");p0(a);}PyErr_Clear();
 #define F(f,i) js(&n,ss(#f));jk(&v,dl(f,i));
  F(setpyerr,1)F(getpyerr,1)F(eval,1)F(e,1)F(py2q,1)F(q2py,1)F(get,1)F(set,2)F(import,1)F(getattr,2)F(call,3)F(isp,1)F(setconv,1)F(getconv,1)js(&n,ss("numpy"));
- jk(&v,kb(!!N));i=1;R xD(n,v);}
+ jk(&v,kb(!!N));t=t1();i=1;R xD(n,v);}
 // a kludge for python modules which try to resolve main e.g. scipy.optimize
 int main(){}
diff --git a/py.h b/py.h
@@ -26,6 +26,7 @@ typedef struct _p _p,*P;struct _p{L r;P t;L n;union{P*p;P v[1];};};typedef struc
  X(V,Py_DecRef,(P))\
  X(V,Py_IncRef,(P))\
  X(V,PyErr_Clear,())\
+ X(V,PyErr_Print,())\
  X(V,PyErr_Fetch,(P*,P*,P*))\
  X(V,PyErr_NormalizeException,(P*,P*,P*))\
  X(P,PyErr_BadArgument,())\
@@ -86,7 +87,6 @@ typedef struct _p _p,*P;struct _p{L r;P t;L n;union{P*p;P v[1];};};typedef struc
  X(P,PyDict_Values,(P))\
  X(P,PyObject_CallFunctionObjArgs,(P,...))\
  X(P,PyImport_Import,(P))\
- X(I,Py_IsInitialized,())\
 
 //https://docs.scipy.org/doc/numpy/reference/c-api.html https://github.com/numpy/numpy/blob/master/numpy/core/code_generators/numpy_api.py
 #undef PyCFunction_New
@@ -111,7 +111,7 @@ ZI pyl(S l){
  HMODULE d=LoadLibrary(l);
 #define X(r,n,a) U(n=(T##n(*)a)GetProcAddress(d,#n))
 #else
- V*d=dlopen(getenv("UNDER_PYTHON")?NULL:l,RTLD_NOW|RTLD_GLOBAL);
+ V*d=dlopen(l,RTLD_NOW|RTLD_GLOBAL);
 #define X(r,n,a) U(n=dlsym(d,#n))
 #endif
  P(!d,0)PF
diff --git a/tests/requirements.txt b/tests/requirements.txt
@@ -1,6 +1,5 @@
 numpy
 beautifulsoup4
 pandas
-tensorflow
 scipy
 matplotlib
diff --git a/tests/tensorflow.t b/tests/tensorflow.t
