Explose memory logging.

Makefile

@@ -0,0 +1,151 @@
+# Makefile for gotch - PyTorch 2.10.0 Go bindings
+# Architecture-specific configuration is loaded from arch/ directory
+
+SHELL := /bin/bash
+
+# Detect OS and architecture
+OS    := $(shell uname -s | tr '[:upper:]' '[:lower:]')
+ARCH  := $(shell uname -m)
+
+# Normalize architecture names
+ifeq ($(ARCH),x86_64)
+	ARCH := amd64
+else ifeq ($(ARCH),aarch64)
+	ARCH := arm64
+endif
+
+# Allow override of architecture configuration
+# Examples:
+#   make ARCH_CONFIG=linux-amd64-cuda build
+#   make ARCH_CONFIG=darwin-arm64 test
+ARCH_CONFIG ?= $(OS)-$(ARCH)
+
+# Include architecture-specific configuration
+ARCH_FILE := arch/$(ARCH_CONFIG).mk
+ifeq ($(wildcard $(ARCH_FILE)),)
+	$(error Architecture config file not found: $(ARCH_FILE). Available: $(wildcard arch/*.mk))
+endif
+include $(ARCH_FILE)
+
+# Go test flags
+TEST_FLAGS := -v
+TEST_TIMEOUT := 5m
+
+.PHONY: all build test test-nn test-ts clean help ffi-validate
+
+# Default target
+all: build
+
+# Build all core packages
+build:
+	@echo "Building gotch core packages..."
+	@go build -v . ./ts ./nn ./vision
+
+# Run all tests
+test: test-nn test-ts ffi-validate
+	@echo "All tests completed"
+
+# Run nn package tests
+# Running with -p 1 to force sequential execution (PyTorch 2.10.0 thread-local gradient state)
+test-nn:
+	@echo "Running nn package tests..."
+	@go test $(TEST_FLAGS) -timeout $(TEST_TIMEOUT) -p 1 -parallel 1 ./nn
+
+# Run ts package tests
+test-ts:
+	@echo "Running ts package tests..."
+	@go test $(TEST_FLAGS) -timeout $(TEST_TIMEOUT) ./ts
+
+# Run specific test in nn package
+# Usage: make test-nn-specific TEST=TestInitTensor_Memcheck
+test-nn-specific:
+	@echo "Running specific nn test: $(TEST)..."
+	@go test $(TEST_FLAGS) -timeout $(TEST_TIMEOUT) -run $(TEST) ./nn
+
+# Run specific test in ts package
+# Usage: make test-ts-specific TEST=TestTensor
+test-ts-specific:
+	@echo "Running specific ts test: $(TEST)..."
+	@go test $(TEST_FLAGS) -timeout $(TEST_TIMEOUT) -run $(TEST) ./ts
+
+# Run tests with coverage
+test-nn-coverage:
+	@echo "Running nn tests with coverage..."
+	@go test -v -timeout $(TEST_TIMEOUT) -coverprofile=coverage-nn.out ./nn
+	@go tool cover -html=coverage-nn.out -o coverage-nn.html
+	@echo "Coverage report saved to coverage-nn.html"
+
+test-ts-coverage:
+	@echo "Running ts tests with coverage..."
+	@go test -v -timeout $(TEST_TIMEOUT) -coverprofile=coverage-ts.out ./ts
+	@go tool cover -html=coverage-ts.out -o coverage-ts.html
+	@echo "Coverage report saved to coverage-ts.html"
+
+# Clean build artifacts
+clean:
+	@echo "Cleaning build artifacts..."
+	@go clean -cache -testcache
+	@rm -f coverage-*.out coverage-*.html
+
+# Display build environment
+env:
+	@echo "Build Environment:"
+	@echo "  Platform:           $(PLATFORM_DESC)"
+	@echo "  Arch Config:        $(ARCH_CONFIG) ($(ARCH_FILE))"
+	@echo "  LIBTORCH_PATH:      $(LIBTORCH_PATH)"
+	@echo "  $(RUNTIME_LIB_VAR): $($(RUNTIME_LIB_VAR))"
+	@echo ""
+	@echo "CGO Flags:"
+	@echo "  CGO_CFLAGS:         $(CGO_CFLAGS)"
+	@echo "  CGO_LDFLAGS:        $(CGO_LDFLAGS)"
+	@echo "  CGO_CXXFLAGS:       $(CGO_CXXFLAGS)"
+	@echo ""
+	@echo "Go version:"
+	@go version
+	@echo ""
+	@echo "LibTorch version: 2.10.0"
+
+# Check if MPS is available
+check-mps:
+	@echo "Checking MPS availability..."
+	@go run -exec 'env DYLD_LIBRARY_PATH=$(DYLD_LIBRARY_PATH)' tools/check_device.go || echo "Create tools/check_device.go to test device availability"
+
+# Validate FFI type conversions
+ffi-validate:
+	@echo "Validating FFI type conversions..."
+	@go run tools/ffi-validation/main.go
+
+# Help target
+help:
+	@echo "Gotch Makefile - PyTorch 2.10.0 Go Bindings"
+	@echo ""
+	@echo "Current Configuration:"
+	@echo "  Platform:      $(PLATFORM_DESC)"
+	@echo "  Arch Config:   $(ARCH_CONFIG)"
+	@echo "  LibTorch:      $(LIBTORCH_PATH)"
+	@echo ""
+	@echo "Targets:"
+	@echo "  make build              - Build all core packages"
+	@echo "  make test               - Run all tests (nn + ts)"
+	@echo "  make test-nn            - Run nn package tests"
+	@echo "  make test-ts            - Run ts package tests"
+	@echo "  make test-nn-specific   - Run specific nn test (TEST=TestName)"
+	@echo "  make test-ts-specific   - Run specific ts test (TEST=TestName)"
+	@echo "  make test-nn-coverage   - Run nn tests with coverage report"
+	@echo "  make test-ts-coverage   - Run ts tests with coverage report"
+	@echo "  make clean              - Clean build artifacts and caches"
+	@echo "  make env                - Display build environment"
+	@echo "  make check-mps          - Check MPS device availability"
+	@echo "  make ffi-validate       - Validate FFI type conversions (C <-> Go)"
+	@echo "  make help               - Show this help message"
+	@echo ""
+	@echo "Architecture Configuration:"
+	@echo "  Default: auto-detected (current: $(ARCH_CONFIG))"
+	@echo "  Override: make ARCH_CONFIG=linux-amd64-cuda build"
+	@echo "  Available configs: $(notdir $(basename $(wildcard arch/*.mk)))"
+	@echo ""
+	@echo "Examples:"
+	@echo "  make test-nn"
+	@echo "  make test-nn-specific TEST=TestInitTensor_Memcheck"
+	@echo "  make ARCH_CONFIG=linux-amd64-cuda build"
+	@echo "  LIBTORCH_PATH=/custom/path make build"

README.md

@@ -20,13 +20,13 @@
 
 ## Dependencies
 
-- **Libtorch** C++ v2.1.0 library of [Pytorch](https://pytorch.org/)
+- **Libtorch** C++ v2.10.0 library of [Pytorch](https://pytorch.org/)
 - Clang-17/Clang++-17 compilers
 
 ## Installation
 
 - Default CUDA version is `11.8` if CUDA is available otherwise using CPU version.
-- Default Pytorch C++ API version is `2.1.0`
+- Default Pytorch C++ API version is `2.10.0`
 
 **NOTE**: `libtorch` will be installed at **`/usr/local/lib`**
 
@@ -266,6 +266,20 @@ func main() {
 
 - See [pkg.go.dev](https://pkg.go.dev/github.com/sugarme/gotch?tab=doc) for APIs detail.
 
+For unit tests use:
+- make test
+
+## PyTorch 2.10.0 Upgrade Notes
+
+This version includes critical fixes for PyTorch 2.10.0 compatibility:
+- Fixed gradient state management (thread-local in PyTorch 2.10.0)
+- Fixed FFI type conversion bugs (C.int → Go int)
+
+To validate FFI conversions:
+```bash
+make ffi-validate
+```
+
 ## License
 
 `gotch` is Apache 2.0 licensed.

arch/README.md

@@ -0,0 +1,108 @@
+# Architecture-Specific Configuration
+
+This directory contains Makefile configuration files for different operating systems and architectures. The main Makefile automatically detects your platform and includes the appropriate configuration file.
+
+## Available Configurations
+
+- **darwin-arm64.mk** - macOS on Apple Silicon (M1/M2/M3)
+- **darwin-amd64.mk** - macOS on Intel processors
+- **linux-amd64.mk** - Linux x86_64 (CPU-only)
+- **linux-amd64-cuda.mk** - Linux x86_64 with CUDA support
+
+## How It Works
+
+1. The main `Makefile` detects your OS and architecture
+2. It includes the corresponding `.mk` file from this directory
+3. The included file sets:
+   - `LIBTORCH_PATH` - Path to LibTorch installation
+   - `CGO_CFLAGS` - C compiler flags for CGO
+   - `CGO_LDFLAGS` - Linker flags for CGO
+   - `CGO_CXXFLAGS` - C++ compiler flags for CGO
+   - Runtime library paths (DYLD_LIBRARY_PATH for macOS, LD_LIBRARY_PATH for Linux)
+
+## Usage
+
+### Default (Auto-Detection)
+
+```bash
+make build
+make test
+```
+
+### Override Architecture
+
+```bash
+# Use CUDA configuration on Linux
+make ARCH_CONFIG=linux-amd64-cuda build
+
+# Use specific configuration
+make ARCH_CONFIG=darwin-arm64 test
+```
+
+### Override LibTorch Path
+
+```bash
+# Temporary override
+LIBTORCH_PATH=/custom/path make build
+
+# Or set in environment
+export LIBTORCH_PATH=/opt/libtorch
+make build
+```
+
+### View Current Configuration
+
+```bash
+make env
+```
+
+## Creating a New Configuration
+
+To add support for a new platform:
+
+1. Create a new `.mk` file in this directory (e.g., `linux-arm64.mk`)
+2. Copy an existing configuration as a template
+3. Modify the paths and flags as needed
+4. Key variables to set:
+   - `LIBTORCH_PATH` - Default path to LibTorch
+   - `RUNTIME_LIB_VAR` - Runtime library path variable name
+   - `CGO_CFLAGS` - C compilation flags
+   - `CGO_LDFLAGS` - Linker flags
+   - `CGO_CXXFLAGS` - C++ flags
+   - `PLATFORM_DESC` - Human-readable platform description
+
+## Configuration Priority
+
+1. Command-line: `LIBTORCH_PATH=/path make build`
+2. Environment: `export LIBTORCH_PATH=/path`
+3. Arch file default: `LIBTORCH_PATH ?= ...` in the `.mk` file
+
+## Example Configuration File
+
+```makefile
+# Architecture configuration for Custom Platform
+
+LIBTORCH_PATH ?= /opt/libtorch
+RUNTIME_LIB_VAR := LD_LIBRARY_PATH
+
+export CGO_CFLAGS := -I$(LIBTORCH_PATH)/include/torch/csrc/api/include -I$(LIBTORCH_PATH)/include
+export CGO_CFLAGS += -O3 -Wall -Wno-deprecated-declarations
+
+export CGO_LDFLAGS := -L$(LIBTORCH_PATH)/lib -ltorch -ltorch_cpu -lc10
+export CGO_LDFLAGS += -Wl,-rpath,$(LIBTORCH_PATH)/lib
+
+export CGO_CXXFLAGS := -std=c++17
+
+export CPATH := $(LIBTORCH_PATH)/include/torch/csrc/api/include:$(LIBTORCH_PATH)/include
+export LIBRARY_PATH := $(LIBTORCH_PATH)/lib
+export LD_LIBRARY_PATH := $(LIBTORCH_PATH)/lib
+
+PLATFORM_DESC := Custom Platform Description
+```
+
+## Notes
+
+- The `libtch/lib.go` file no longer contains hardcoded paths
+- All CGO configuration is managed through the Makefile and these arch files
+- This allows the same codebase to work across different platforms without code changes
+- Always use `make build` instead of `go build` directly to ensure CGO flags are properly set

arch/darwin-amd64.mk

@@ -0,0 +1,26 @@
+# Architecture configuration for macOS x86_64 (Intel)
+# This file is included by the main Makefile based on detected OS and architecture
+
+# LibTorch installation path (can be overridden via LIBTORCH environment variable)
+LIBTORCH_PATH ?= $(shell [ -d "$(HOME)/src/gotch/libtorch-2.10.0-macos" ] && echo "$(HOME)/src/gotch/libtorch-2.10.0-macos" || echo "$(CURDIR)/libtorch-2.10.0-macos")
+
+# Runtime library path variable for macOS
+RUNTIME_LIB_VAR := DYLD_LIBRARY_PATH
+
+# CGO flags for compilation
+export CGO_CFLAGS := -I$(LIBTORCH_PATH)/include/torch/csrc/api/include -I$(LIBTORCH_PATH)/include
+export CGO_CFLAGS += -O3 -Wall -Wno-unused-variable -Wno-deprecated-declarations -Wno-c++11-narrowing -g -Wno-sign-compare -Wno-unused-function
+
+export CGO_LDFLAGS := -L$(LIBTORCH_PATH)/lib -ltorch -ltorch_cpu -lc10
+export CGO_LDFLAGS += -Wl,-rpath,$(LIBTORCH_PATH)/lib
+
+export CGO_CXXFLAGS := -std=c++17 -I$(LIBTORCH_PATH)/include/torch/csrc/api/include -I$(LIBTORCH_PATH)/include
+export CGO_CXXFLAGS += -O3 -Wall -Wno-unused-variable -Wno-deprecated-declarations -Wno-c++11-narrowing -g -Wno-sign-compare -Wno-unused-function
+
+# Environment for runtime (tests, etc.)
+export CPATH := $(LIBTORCH_PATH)/include/torch/csrc/api/include:$(LIBTORCH_PATH)/include
+export LIBRARY_PATH := $(LIBTORCH_PATH)/lib
+export DYLD_LIBRARY_PATH := $(LIBTORCH_PATH)/lib
+
+# Platform-specific notes
+PLATFORM_DESC := macOS x86_64 (Intel)

arch/darwin-arm64.mk

@@ -0,0 +1,26 @@
+# Architecture configuration for macOS ARM64 (Apple Silicon)
+# This file is included by the main Makefile based on detected OS and architecture
+
+# LibTorch installation path (can be overridden via LIBTORCH environment variable)
+LIBTORCH_PATH ?= $(shell [ -d "$(HOME)/src/gotch/libtorch-2.10.0-macos" ] && echo "$(HOME)/src/gotch/libtorch-2.10.0-macos" || echo "$(CURDIR)/libtorch-2.10.0-macos")
+
+# Runtime library path variable for macOS
+RUNTIME_LIB_VAR := DYLD_LIBRARY_PATH
+
+# CGO flags for compilation
+export CGO_CFLAGS := -I$(LIBTORCH_PATH)/include/torch/csrc/api/include -I$(LIBTORCH_PATH)/include
+export CGO_CFLAGS += -O3 -Wall -Wno-unused-variable -Wno-deprecated-declarations -Wno-c++11-narrowing -g -Wno-sign-compare -Wno-unused-function
+
+export CGO_LDFLAGS := -L$(LIBTORCH_PATH)/lib -ltorch -ltorch_cpu -lc10
+export CGO_LDFLAGS += -Wl,-rpath,$(LIBTORCH_PATH)/lib
+
+export CGO_CXXFLAGS := -std=c++17 -I$(LIBTORCH_PATH)/include/torch/csrc/api/include -I$(LIBTORCH_PATH)/include
+export CGO_CXXFLAGS += -O3 -Wall -Wno-unused-variable -Wno-deprecated-declarations -Wno-c++11-narrowing -g -Wno-sign-compare -Wno-unused-function
+
+# Environment for runtime (tests, etc.)
+export CPATH := $(LIBTORCH_PATH)/include/torch/csrc/api/include:$(LIBTORCH_PATH)/include
+export LIBRARY_PATH := $(LIBTORCH_PATH)/lib
+export DYLD_LIBRARY_PATH := $(LIBTORCH_PATH)/lib
+
+# Platform-specific notes
+PLATFORM_DESC := macOS ARM64 (Apple Silicon) with MPS support

arch/linux-amd64-cuda.mk

@@ -0,0 +1,26 @@
+# Architecture configuration for Linux x86_64 with CUDA
+# This file is included by the main Makefile based on detected OS and architecture
+
+# LibTorch installation path (can be overridden via LIBTORCH environment variable)
+LIBTORCH_PATH ?= $(shell [ -d "/opt/libtorch-cuda" ] && echo "/opt/libtorch-cuda" || echo "$(CURDIR)/libtorch")
+
+# Runtime library path variable for Linux
+RUNTIME_LIB_VAR := LD_LIBRARY_PATH
+
+# CGO flags for compilation
+export CGO_CFLAGS := -I$(LIBTORCH_PATH)/include/torch/csrc/api/include -I$(LIBTORCH_PATH)/include
+export CGO_CFLAGS += -O3 -Wall -Wno-unused-variable -Wno-deprecated-declarations -Wno-c++11-narrowing -g -Wno-sign-compare -Wno-unused-function
+
+export CGO_LDFLAGS := -L$(LIBTORCH_PATH)/lib -ltorch -ltorch_cpu -ltorch_cuda -lc10 -lc10_cuda
+export CGO_LDFLAGS += -Wl,-rpath,$(LIBTORCH_PATH)/lib
+
+export CGO_CXXFLAGS := -std=c++17 -I$(LIBTORCH_PATH)/include/torch/csrc/api/include -I$(LIBTORCH_PATH)/include
+export CGO_CXXFLAGS += -O3 -Wall -Wno-unused-variable -Wno-deprecated-declarations -Wno-c++11-narrowing -g -Wno-sign-compare -Wno-unused-function
+
+# Environment for runtime (tests, etc.)
+export CPATH := $(LIBTORCH_PATH)/include/torch/csrc/api/include:$(LIBTORCH_PATH)/include
+export LIBRARY_PATH := $(LIBTORCH_PATH)/lib
+export LD_LIBRARY_PATH := $(LIBTORCH_PATH)/lib
+
+# Platform-specific notes
+PLATFORM_DESC := Linux x86_64 with CUDA support