diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3e45a134ed..7649c92050 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -165,9 +165,9 @@ else()
   message(STATUS "Could not find ccache. Consider installing ccache to speed up compilation.")
 endif()
 
-add_subdirectory(third_party)
 add_subdirectory(torchaudio/csrc)
 if (BUILD_SOX)
+  add_subdirectory(third_party/sox)
   add_subdirectory(torchaudio/csrc/sox)
 endif()
 if (USE_FFMPEG)
diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt
deleted file mode 100644
index 07cd3c9d4b..0000000000
--- a/third_party/CMakeLists.txt
+++ /dev/null
@@ -1,11 +0,0 @@
-set(CMAKE_CXX_VISIBILITY_PRESET hidden)
-
-file(MAKE_DIRECTORY install/include)
-file(MAKE_DIRECTORY install/lib)
-
-################################################################################
-# sox
-################################################################################
-if (BUILD_SOX)
-  add_subdirectory(sox)
-endif()