futurile/guix-play
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

gnu: Add python-pytorch.

Browse Source 
* gnu/packages/machine-learning.scm (python-pytorch): New variable.
* gnu/packages/patches/python-pytorch-runpath.patch,
gnu/packages/patches/python-pytorch-system-libraries.patch: New files.
* gnu/local.mk (dist_patch_DATA): Add them.
This commit is contained in:
Ludovic Courtès 2021-08-04 16:15:17 +02:00 committed by Ludovic Courtès
parent 38d319d38a
commit dd55ad7ff8
No known key found for this signature in database
GPG Key ID: 090B11993D9AEBB5
4 changed files with 300 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
gnu/local.mk
View File

@ -1649,6 +1649,8 @@ dist_patch_DATA = \
%D%/packages/patches/python-pydot-regression-test.patch \
%D%/packages/patches/python2-pygobject-2-deprecation.patch \
%D%/packages/patches/python-pygpgme-fix-pinentry-tests.patch \
%D%/packages/patches/python-pytorch-runpath.patch \
%D%/packages/patches/python-pytorch-system-libraries.patch \
%D%/packages/patches/python-robotframework-source-date-epoch.patch \
%D%/packages/patches/python-seaborn-kde-test.patch \
%D%/packages/patches/python2-subprocess32-disable-input-test.patch \

143
gnu/packages/machine-learning.scm
View File

@ -65,10 +65,12 @@
#:use-module (gnu packages graphviz)
#:use-module (gnu packages gstreamer)
#:use-module (gnu packages image)
#:use-module (gnu packages libffi)
#:use-module (gnu packages linux)
#:use-module (gnu packages llvm)
#:use-module (gnu packages maths)
#:use-module (gnu packages mpi)
#:use-module (gnu packages ninja)
#:use-module (gnu packages ocaml)
#:use-module (gnu packages onc-rpc)
#:use-module (gnu packages parallel)
@ -2546,7 +2548,7 @@ general non-linear dimension reduction.")
(package
(name "xnnpack")
(version (git-version version revision commit))
(home-page "https://github.com/google/XNNPACK")
(home-page "https://github.com/google/XNNPACK") ;fork of QNNPACK
(source (origin
(method git-fetch)
(uri (git-reference (url home-page) (commit commit)))
@ -2586,3 +2588,142 @@ high-level machine learning frameworks, such as TensorFlow Lite,
TensorFlow.js, PyTorch, and MediaPipe.")
(license license:bsd-3))))
(define-public python-pytorch
(package
(name "python-pytorch")
(version "1.9.0")
(source (origin
(method git-fetch)
(uri (git-reference
(url "https://github.com/pytorch/pytorch")
(commit (string-append "v" version))
(recursive? #t)))
(file-name (git-file-name name version))
(sha256
(base32
"0cznsh68hwk5761gv7iijb4g6jgjpvs3bbixwpzzmkbkbn2q96c1"))
(patches (search-patches "python-pytorch-system-libraries.patch"
"python-pytorch-runpath.patch"))
(modules '((guix build utils)))
(snippet
'(begin
;; XXX: Let's be clear: this package is a bundling fest. We
;; delete as much as we can, but there's still a lot left.
(for-each (lambda (directory)
(delete-file-recursively
(string-append "third_party/" directory)))
'("benchmark" "cpuinfo" "eigen"
;; FIXME: QNNPACK (of which XNNPACK is a fork)
;; needs these.
;; "FP16" "FXdiv" "gemmlowp" "psimd"
"gloo" "googletest" "ios-cmake"
"onnx" "protobuf" "pthreadpool"
"pybind11" "python-enum" "python-peachpy"
"python-six" "tbb" "XNNPACK" "zstd"))
;; Adjust references to the onnx-optimizer headers.
(substitute* "caffe2/onnx/backend.cc"
(("onnx/optimizer/")
"onnxoptimizer/"))))))
(build-system python-build-system)
(arguments
'(#:phases (modify-phases %standard-phases
(add-before 'build 'use-system-libraries
(lambda* (#:key outputs #:allow-other-keys)
;; Tell 'setup.py' to let 'CMakeLists.txt' know that we
;; want to use "system libraries" instead of the bundled
;; ones.
(setenv "USE_SYSTEM_LIBS" "1")
;; XXX: Disable that for simplicity for now.
(setenv "USE_FBGEMM" "0")
;; Set 'OUT/lib' rather than '$ORIGIN' as the RUNPATH on
;; binaries that go to 'lib/python3.8/torch/bin' & co.
#;(substitute* "cmake/Dependencies.cmake"
(("^set\\(CMAKE_INSTALL_RPATH .*")
(string-append "set(CMAKE_INSTALL_RPATH \""
(assoc-ref outputs "out")
"/lib\")\n")))))
(add-before 'build 'make-things-writable
(lambda _
;; The 'build_caffe2' function in
;; 'tools/build_pytorch_libs.py', called from the
;; top-level 'setup.py', needs write access to this
;; directory.
(for-each make-file-writable
(find-files "caffe2/proto" "."
#:directories? #t))))
(replace 'check
(lambda* (#:key inputs outputs tests? #:allow-other-keys)
;; Run the test suite following the instructions in
;; 'CONTRIBUTING.md'. XXX: Unfortunately this doesn't
;; work, unless you set PYTHONPATH presumably.
(when tests?
(let ((python-site (site-packages inputs outputs)))
(setenv "PYTHONPATH"
(string-append python-site ":"
(getenv "PYTHONPATH")))
(invoke "python" "test/run_test.py")))))
(add-after 'install 'remove-test-executables
(lambda* (#:key inputs outputs #:allow-other-keys)
;; Remove test executables, but keep other executables
;; such as 'torch_shm_manager' and and .so files such as
;; 'libtorch_global_deps.so'.
(let ((python-site (site-packages inputs outputs)))
(for-each delete-file
(find-files python-site
"(^test_cpp_rpc|_test)$"))))))
;; XXX: Tests attempt to download data such as
;; <https://raw.githubusercontent.com/pytorch/test-infra/master/stats/slow-tests.json>.
#:tests? #f))
(native-inputs
`(("cmake" ,cmake)
("ninja" ,ninja)))
(inputs
`(("eigen" ,eigen)
;; ("fmt" ,fmt)
("fp16" ,fp16)
("gemmlowp" ,gemmlowp)
("googletest" ,googletest)
("googlebenchmark" ,googlebenchmark)
("gloo" ,gloo)
("openblas" ,openblas)
("openmpi" ,openmpi)
("pthreadpool" ,pthreadpool)
("protobuf" ,protobuf)
("pybind11" ,pybind11)
("sleef" ,sleef)
("xnnpack" ,xnnpack)
("zstd" ,zstd)))
(propagated-inputs
`(("python-astunparse" ,python-astunparse)
("python-numpy" ,python-numpy)
("python-pyyaml" ,python-pyyaml)
("python-cffi" ,python-cffi)
("python-peachpy" ,python-peachpy)
("python-typing-extensions" ,python-typing-extensions)
("python-future" ,python-future)
("python-six" ,python-six)
("python-requests" ,python-requests)
("onnx" ,onnx) ;propagated for its Python modules
("onnx-optimizer" ,onnx-optimizer)
("cpuinfo" ,cpuinfo)))
(home-page "https://pytorch.org/")
(synopsis "Python library for tensor computation and deep neural networks")
(description
"PyTorch is a Python package that provides two high-level features:
@itemize
@item tensor computation (like NumPy) with strong GPU acceleration;
@item deep neural networks (DNNs) built on a tape-based autograd system.
@end itemize
You can reuse Python packages such as NumPy, SciPy, and Cython to extend
PyTorch when needed.
Note: currently this package does not provide GPU support.")
(license license:bsd-3)))

25
gnu/packages/patches/python-pytorch-runpath.patch Normal file
View File

@ -0,0 +1,25 @@
Libraries (such as 'libtorch_cpu.so') and executables (such as 'torch_shm_manager')
get installed, quite surprisingly, to 'lib/python3.8/site-packages/{bin,lib}'.
Make sure RUNPATH matches that.
--- a/cmake/Dependencies.cmake
+++ b/cmake/Dependencies.cmake
@@ -4,7 +4,7 @@ if(APPLE)
set(CMAKE_MACOSX_RPATH ON)
set(_rpath_portable_origin "@loader_path")
else()
- set(_rpath_portable_origin $ORIGIN)
+ set(_rpath_portable_origin $ORIGIN/../lib)
endif(APPLE)
# Use separate rpaths during build and install phases
set(CMAKE_SKIP_BUILD_RPATH FALSE)
--- a/caffe2/CMakeLists.txt
+++ b/caffe2/CMakeLists.txt
@@ -1797,5 +1797,5 @@ if(BUILD_PYTHON)
if(${BUILDING_WITH_TORCH_LIBS})
# site-packages/caffe2/python/caffe2_pybind11_state
# site-packages/torch/lib
- set(caffe2_pybind11_rpath "${_rpath_portable_origin}/../../torch/lib")
+ set(caffe2_pybind11_rpath $ORIGIN/../../torch/lib)
endif(${BUILDING_WITH_TORCH_LIBS})

131
gnu/packages/patches/python-pytorch-system-libraries.patch Normal file
View File

@ -0,0 +1,131 @@
Use our own googletest rather than the bundled one.
Get NNPACK to use our own PeachPy rather than the bundled one.
diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake
index 5d57b9ca78..620cca4e60 100644
--- a/cmake/Dependencies.cmake
+++ b/cmake/Dependencies.cmake
@@ -644,11 +644,6 @@ if(BUILD_TEST OR BUILD_MOBILE_BENCHMARK OR BUILD_MOBILE_TEST)
# this shouldn't be necessary anymore.
get_property(INC_DIR_temp DIRECTORY PROPERTY INCLUDE_DIRECTORIES)
set_property(DIRECTORY PROPERTY INCLUDE_DIRECTORIES "")
- add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/../third_party/googletest)
- set_property(DIRECTORY PROPERTY INCLUDE_DIRECTORIES ${INC_DIR_temp})
-
- include_directories(BEFORE SYSTEM ${CMAKE_CURRENT_LIST_DIR}/../third_party/googletest/googletest/include)
- include_directories(BEFORE SYSTEM ${CMAKE_CURRENT_LIST_DIR}/../third_party/googletest/googlemock/include)
# We will not need to test benchmark lib itself.
set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "Disable benchmark testing as we don't need it.")
@@ -1485,7 +1480,7 @@ if(CAFFE2_CMAKE_BUILDING_WITH_MAIN_REPO AND NOT INTERN_DISABLE_ONNX)
endif()
set_property(TARGET onnx_proto PROPERTY IMPORTED_LOCATION ${ONNX_PROTO_LIBRARY})
message("-- Found onnx: ${ONNX_LIBRARY} ${ONNX_PROTO_LIBRARY}")
- list(APPEND Caffe2_DEPENDENCY_LIBS onnx_proto onnx)
+ list(APPEND Caffe2_DEPENDENCY_LIBS onnx_proto onnx onnx_optimizer)
endif()
include_directories(${FOXI_INCLUDE_DIRS})
list(APPEND Caffe2_DEPENDENCY_LIBS foxi_loader)
diff --git a/caffe2/CMakeLists.txt b/caffe2/CMakeLists.txt
index 50ebb224ce..5953d9ddf7 100644
--- a/caffe2/CMakeLists.txt
+++ b/caffe2/CMakeLists.txt
@@ -1632,7 +1632,7 @@ if(BUILD_TEST)
if(NOT MSVC)
add_executable(${test_name}_${CPU_CAPABILITY} "${test_src}" ../aten/src/ATen/native/quantized/affine_quantizer_base.cpp)
# TODO: Get rid of c10 dependency (which is only needed for the implementation of AT_ERROR)
- target_link_libraries(${test_name}_${CPU_CAPABILITY} c10 sleef gtest_main)
+ target_link_libraries(${test_name}_${CPU_CAPABILITY} c10 sleef gtest_main gtest)
if(USE_FBGEMM)
target_link_libraries(${test_name}_${CPU_CAPABILITY} fbgemm)
endif()
@@ -1655,7 +1655,7 @@ if(BUILD_TEST)
foreach(test_src ${Caffe2_CPU_TEST_SRCS})
get_filename_component(test_name ${test_src} NAME_WE)
add_executable(${test_name} "${test_src}")
- target_link_libraries(${test_name} torch_library gtest_main)
+ target_link_libraries(${test_name} torch_library gtest_main gtest)
target_include_directories(${test_name} PRIVATE $<INSTALL_INTERFACE:include>)
target_include_directories(${test_name} PRIVATE $<BUILD_INTERFACE:${CMAKE_BINARY_DIR}/include>)
target_include_directories(${test_name} PRIVATE ${Caffe2_CPU_INCLUDE})
@@ -1673,7 +1673,7 @@ if(BUILD_TEST)
foreach(test_src ${Caffe2_GPU_TEST_SRCS})
get_filename_component(test_name ${test_src} NAME_WE)
cuda_add_executable(${test_name} "${test_src}")
- target_link_libraries(${test_name} torch_library gtest_main)
+ target_link_libraries(${test_name} torch_library gtest_main gtest)
target_include_directories(${test_name} PRIVATE $<INSTALL_INTERFACE:include>)
target_include_directories(${test_name} PRIVATE ${Caffe2_CPU_INCLUDE})
add_test(NAME ${test_name} COMMAND $<TARGET_FILE:${test_name}>)
@@ -1691,7 +1691,7 @@ if(BUILD_TEST)
foreach(test_src ${Caffe2_VULKAN_TEST_SRCS})
get_filename_component(test_name ${test_src} NAME_WE)
add_executable(${test_name} "${test_src}")
- target_link_libraries(${test_name} torch_library gtest_main)
+ target_link_libraries(${test_name} torch_library gtest_main gtest)
target_include_directories(${test_name} PRIVATE $<INSTALL_INTERFACE:include>)
target_include_directories(${test_name} PRIVATE ${Caffe2_CPU_INCLUDE})
add_test(NAME ${test_name} COMMAND $<TARGET_FILE:${test_name}>)
@@ -1709,7 +1709,7 @@ if(BUILD_TEST)
foreach(test_src ${Caffe2_HIP_TEST_SRCS})
get_filename_component(test_name ${test_src} NAME_WE)
add_executable(${test_name} "${test_src}")
- target_link_libraries(${test_name} torch_library gtest_main)
+ target_link_libraries(${test_name} torch_library gtest_main gtest)
target_include_directories(${test_name} PRIVATE $<INSTALL_INTERFACE:include>)
target_include_directories(${test_name} PRIVATE ${Caffe2_CPU_INCLUDE} ${Caffe2_HIP_INCLUDE})
target_compile_options(${test_name} PRIVATE ${HIP_CXX_FLAGS})
diff --git a/torch/lib/c10d/test/CMakeLists.txt b/torch/lib/c10d/test/CMakeLists.txt
index b74d4b65f7..fc7c207505 100644
--- a/torch/lib/c10d/test/CMakeLists.txt
+++ b/torch/lib/c10d/test/CMakeLists.txt
@@ -16,25 +16,25 @@ function(c10d_add_test test_src)
add_test(NAME ${test_name} COMMAND $<TARGET_FILE:${test_name}>)
endfunction()
-c10d_add_test(FileStoreTest.cpp c10d gtest_main)
-c10d_add_test(TCPStoreTest.cpp c10d gtest_main)
+c10d_add_test(FileStoreTest.cpp c10d gtest_main gtest)
+c10d_add_test(TCPStoreTest.cpp c10d gtest_main gtest)
if(NOT WIN32)
- c10d_add_test(HashStoreTest.cpp c10d gtest_main)
+ c10d_add_test(HashStoreTest.cpp c10d gtest_main gtest)
endif()
if(USE_CUDA)
if(USE_C10D_GLOO)
- c10d_add_test(ProcessGroupGlooTest.cpp c10d c10d_cuda_test gtest_main)
- c10d_add_test(ProcessGroupGlooAsyncTest.cpp c10d c10d_cuda_test gtest_main)
+ c10d_add_test(ProcessGroupGlooTest.cpp c10d c10d_cuda_test gtest_main gtest)
+ c10d_add_test(ProcessGroupGlooAsyncTest.cpp c10d c10d_cuda_test gtest_main gtest)
endif()
if(USE_C10D_NCCL)
- c10d_add_test(ProcessGroupNCCLTest.cpp c10d c10d_cuda_test gtest_main)
+ c10d_add_test(ProcessGroupNCCLTest.cpp c10d c10d_cuda_test gtest_main gtest)
c10d_add_test(ProcessGroupNCCLErrorsTest.cpp c10d c10d_cuda_test
- gtest_main)
+ gtest_main gtest)
endif()
else()
if(USE_C10D_GLOO)
- c10d_add_test(ProcessGroupGlooTest.cpp c10d gtest_main)
+ c10d_add_test(ProcessGroupGlooTest.cpp c10d gtest_main gtest)
endif()
endif()
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5ecd2df..24feae3 100644
--- a/third_party/NNPACK/CMakeLists.txt
+++ b/third_party/NNPACK/CMakeLists.txt
@@ -427,8 +427,7 @@ IF(NNPACK_BACKEND STREQUAL "x86-64")
FILE(MAKE_DIRECTORY ${obj_dir})
ADD_CUSTOM_COMMAND(
OUTPUT ${obj}
- COMMAND "PYTHONPATH=${PEACHPY_PYTHONPATH}"
- ${PYTHON_EXECUTABLE} -m peachpy.x86_64
+ COMMAND ${PYTHON_EXECUTABLE} -m peachpy.x86_64
-mabi=sysv -g4 -mimage-format=${PEACHPY_IMAGE_FORMAT}
"-I${PROJECT_SOURCE_DIR}/src" "-I${PROJECT_SOURCE_DIR}/src/x86_64-fma" "-I${FP16_SOURCE_DIR}/include"
-o ${obj} "${PROJECT_SOURCE_DIR}/${src}"