--- a/setup.py
+++ b/setup.py
@@ -473,14 +473,6 @@ def build_deps():
     check_pydep("yaml", "pyyaml")
     build_python = not BUILD_LIBTORCH_WHL
 
-    build_caffe2(
-        version=version,
-        cmake_python_library=cmake_python_library,
-        build_python=build_python,
-        rerun_cmake=RERUN_CMAKE,
-        cmake_only=CMAKE_ONLY,
-        cmake=cmake,
-    )
 
     if CMAKE_ONLY:
         report(