Initial support for scripting (#4016)

* Add ZeroMQ external submodule

* ZeroMQ libzmq building on macOS

* Added RPC namespace, settings and logging

* Added request queue handling and new classes

* Add C++ interface to ZeroMQ

* Added start of ZeroMQ RPC Server implementation.

* Request construction and callback request handling

* Read and write memory implementation

* Add ID to request format and send reply

* Add RPC setting to macOS UI

* Fixed initialization order bug and added exception handling

* Working read-write through Python

* Update CMakeLists for libzmq to resolve target name conflict on Windows

* Platform-specific CMake definitions for Windows/non-Windows

* Add comments

* Revert "Add RPC setting to macOS UI"

* Always run RPC server instead of configurable

* Add Python scripting example. Updated .gitignore

* Rename member variables to remove trailing underscore

* Finally got libzmq external project building on macOS

* Add missing dependency during libzmq build

* Adding more missing dependencies [skip ci]

* Only build what is required from libzmq

* Extra length checks on client input

* Call InvalidateCacheRange after memory write

* Revert MinGW change. Fix clang-format. Improve error handling in request/reply. Allow any length of data read/write in Python.

* Re-organized RPC static global state into a proper class. [skip ci]

* Make sure libzmq always builds in Release mode

* Renamed Request to Packet since Request and Reply are the same thing

* Moved request fulfillment out of Packet and into RPCServer

* Change request thread from sleep to condition variable

* Remove non-blocking polling from ZMQ server code. Receive now blocks and terminates properly without sleeping. This change significantly improves script speed.

* Move scripting files to dist/ instead of src/

* C++ code review changes for jroweboy [skip ci]

* Python code review changes for jroweboy [skip ci]

* Add docstrings and tests to citra.py [skip ci]

* Add host OS check for libzmq build

* Revert "Add host OS check for libzmq build"

* Fixed a hang when emulation is stopped and restarted due to improper destruction order of ZMQ objects [skip ci]

* Add scripting directory to archive packaging [skip ci]

* Specify C/CXX compiler variables on MinGW build

* Only specify compiler on Linux mingw

* Use gcc and g++ on Windows mingw

* Specify generator for mingw

* Don't specify toolchain on windows mingw

* Changed citra.py to support Python 3 instead of Python 2

* Fix bug where RPC wouldn't restart after Stop/Start emulation

* Added copyright to headers and reorganized includes and forward declarations

externals/CMakeLists.txt

@@ -2,6 +2,7 @@
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${PROJECT_SOURCE_DIR}/CMakeModules)
 include(DownloadExternals)
+include(ExternalProject)
 
 # Catch
 add_library(catch-single-include INTERFACE)
@@ -92,3 +93,96 @@ if (ENABLE_WEB_SERVICE)
     add_library(json-headers INTERFACE)
     target_include_directories(json-headers INTERFACE ./json)
 endif()
+
+# ZeroMQ
+# libzmq includes its own clang-format target, which conflicts with the
+# clang-format in Citra if libzmq is added as a subdirectory. An external
+# project gets around this issue. Unfortunately, a lot of different
+# configuration options are required for each different platform. An
+# attempt was made to use CMake variables where possible, but some
+# information necessarily had to be repeated. Hopefully there is not
+# often a need to change anything.
+if (MINGW)
+    if (${CMAKE_HOST_SYSTEM_NAME} STREQUAL "Windows")
+        set(LIBZMQ_MAKE mingw32-make)
+        set(LIBZMQ_COMPILER "")
+        set(LIBZMQ_TOOLCHAIN_FILE "")
+    else()
+        set(LIBZMQ_MAKE make)
+        set(LIBZMQ_COMPILER -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER};-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER})
+        set(LIBZMQ_TOOLCHAIN_FILE -DCMAKE_TOOLCHAIN_FILE=${PROJECT_SOURCE_DIR}/CMakeModules/MinGWCross.cmake)
+    endif()
+    ExternalProject_Add(libzmq-external
+        SOURCE_DIR ./libzmq
+        CMAKE_ARGS -DWITH_PERF_TOOL=OFF;-DZMQ_BUILD_TESTS=OFF;-DENABLE_CPACK=OFF;-DCMAKE_MAKE_PROGRAM=${LIBZMQ_MAKE};-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE};${LIBZMQ_TOOLCHAIN_FILE};${LIBZMQ_COMPILER}
+        BUILD_COMMAND cmake --build ${CMAKE_CURRENT_BINARY_DIR}/libzmq-external-prefix/src/libzmq-external-build --target libzmq-static --config ${CMAKE_BUILD_TYPE}
+        GIT_REPOSITORY https://github.com/zeromq/libzmq
+        GIT_TAG v4.2.5
+        INSTALL_COMMAND "")
+else()
+    if (MSVC)
+        set(LIBZMQ_COMPILER_FLAGS -DCMAKE_C_FLAGS=/GL-;-DCMAKE_CXX_FLAGS=/GL-)
+    else()
+        set(LIBZMQ_COMPILER_FLAGS "")
+    endif()
+    ExternalProject_Add(libzmq-external
+        SOURCE_DIR ./libzmq
+        CMAKE_ARGS -DCMAKE_MACOSX_RPATH=1;-DCMAKE_OSX_ARCHITECTURES=x86_64;-DWITH_PERF_TOOL=OFF;-DZMQ_BUILD_TESTS=OFF;-DENABLE_CPACK=OFF;-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE};${LIBZMQ_COMPILER_FLAGS}
+        BUILD_COMMAND cmake --build ${CMAKE_CURRENT_BINARY_DIR}/libzmq-external-prefix/src/libzmq-external-build --target libzmq-static --config ${CMAKE_BUILD_TYPE}
+        GIT_REPOSITORY https://github.com/zeromq/libzmq
+        GIT_TAG v4.2.5
+        INSTALL_COMMAND "")
+endif()
+set(LIBZMQ_DIR ${CMAKE_CURRENT_BINARY_DIR}/libzmq-external-prefix/src/libzmq-external-build/lib)
+# On macOS, we need to build a fat static library containing both x86_64 and x86_64h, since macOS
+# targets specify two architectures in their link command line ("-arch x86_64 -arch x86_64h").
+if (APPLE)
+    ExternalProject_Add(libzmq-external-h
+        SOURCE_DIR ./libzmq-h
+        CMAKE_ARGS -DCMAKE_MACOSX_RPATH=1;-DCMAKE_OSX_ARCHITECTURES=x86_64h;-DWITH_PERF_TOOL=OFF;-DZMQ_BUILD_TESTS=OFF;-DENABLE_CPACK=OFF;-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+        BUILD_COMMAND cmake --build ${CMAKE_CURRENT_BINARY_DIR}/libzmq-external-h-prefix/src/libzmq-external-h-build --target libzmq-static --config ${CMAKE_BUILD_TYPE}
+        GIT_REPOSITORY https://github.com/zeromq/libzmq
+        GIT_TAG v4.2.5
+        INSTALL_COMMAND "")
+    set(LIBZMQ_H_DIR ${CMAKE_CURRENT_BINARY_DIR}/libzmq-external-h-prefix/src/libzmq-external-h-build/lib)
+
+    add_library(libzmq-external-imported STATIC IMPORTED GLOBAL)
+    add_library(libzmq-external-imported-h STATIC IMPORTED GLOBAL)
+    add_dependencies(libzmq-external-imported libzmq-external)
+    add_dependencies(libzmq-external-imported-h libzmq-external-h)
+else()
+    add_library(libzmq STATIC IMPORTED GLOBAL)
+    add_dependencies(libzmq libzmq-external)
+endif()
+# Set up the imported target properties
+if (MSVC)
+    set_target_properties(libzmq PROPERTIES IMPORTED_LOCATION ${LIBZMQ_DIR}/${CMAKE_BUILD_TYPE}/libzmq-v141-mt-s-4_2_5${CMAKE_STATIC_LIBRARY_SUFFIX})
+    set_target_properties(libzmq PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES iphlpapi${CMAKE_STATIC_LIBRARY_SUFFIX})
+else()
+    if (APPLE)
+        set_target_properties(libzmq-external-imported PROPERTIES IMPORTED_LOCATION ${LIBZMQ_DIR}/libzmq${CMAKE_STATIC_LIBRARY_SUFFIX})
+        set_target_properties(libzmq-external-imported-h PROPERTIES IMPORTED_LOCATION ${LIBZMQ_H_DIR}/libzmq${CMAKE_STATIC_LIBRARY_SUFFIX})
+    else()
+        set_target_properties(libzmq PROPERTIES IMPORTED_LOCATION ${LIBZMQ_DIR}/libzmq${CMAKE_STATIC_LIBRARY_SUFFIX})
+        if(MINGW)
+            set_target_properties(libzmq PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES "ws2_32${CMAKE_STATIC_LIBRARY_SUFFIX};iphlpapi${CMAKE_STATIC_LIBRARY_SUFFIX}")
+        endif()
+    endif()
+endif()
+# On macOS, create the combined target
+if (APPLE)
+    set(LIBZMQ_COMBINED_OUTPUT ${LIBZMQ_DIR}/libzmq_combined${CMAKE_STATIC_LIBRARY_SUFFIX})
+    add_custom_target(libzmq-combined COMMAND lipo -create ${LIBZMQ_DIR}/libzmq${CMAKE_STATIC_LIBRARY_SUFFIX} ${LIBZMQ_H_DIR}/libzmq${CMAKE_STATIC_LIBRARY_SUFFIX} -o ${LIBZMQ_COMBINED_OUTPUT}
+                                      BYPRODUCTS ${LIBZMQ_COMBINED_OUTPUT})
+    add_dependencies(libzmq-combined libzmq-external-imported libzmq-external-imported-h)
+    add_library(libzmq STATIC IMPORTED GLOBAL)
+    set_target_properties(libzmq PROPERTIES IMPORTED_LOCATION ${LIBZMQ_COMBINED_OUTPUT})
+    add_dependencies(libzmq libzmq-combined)
+endif()
+# C interface to ZeroMQ
+add_library(libzmq-headers INTERFACE)
+target_include_directories(libzmq-headers INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/libzmq/include)
+# C++ interface to ZeroMQ
+add_library(cppzmq-headers INTERFACE)
+target_include_directories(cppzmq-headers INTERFACE ./cppzmq)
+add_dependencies(cppzmq-headers libzmq)