diff --git a/Intermediate/bim_slimmer.csx b/Intermediate/bim_slimmer.csx new file mode 100644 index 0000000..4b6fbb0 --- /dev/null +++ b/Intermediate/bim_slimmer.csx @@ -0,0 +1,253 @@ +// +// Title: BIM Slimmer - strip metadata bloat to reduce file size +// +// Author: Alexis Olson using GPT-5 Thinking and Claude Opus 4.1 +// +// Description: +// Opens a Tabular model .bim file, removes UI/engine bloat while preserving model semantics, +// and saves a sibling .slim JSON. Preserves structural containers (model, tables, columns, +// measures, partitions, relationships, etc.), removes empty/unused values, and supports +// optional switches for display/query group metadata. Shows a summary with items removed and +// size savings. +// +// How to use: +// - Use in Tabular Editor (2 or 3) Advanced Scripting. +// - (Optional) Customize the configuration options at the top of the script. +// - When prompted, select a .bim file. +// - The script writes .slim and displays a summary. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Windows.Forms; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +// ==================== CONFIGURATION ==================== +// Use defaults specified or customize as needed + +// Core metadata removal (recommended: ON) +bool REMOVE_Annotations = true; // annotations, changedProperties, extendedProperties +bool REMOVE_Lineage = true; // lineageTag, sourceLineageTag +bool REMOVE_LanguageData = true; // cultures, translations, synonyms, linguisticMetadata + +// Value-based cleanup (recommended: ON) +bool REMOVE_DefaultValues = true; // dataCategory:Uncategorized, summarizeBy:none +bool REMOVE_RedundantNames = true; // sourceColumn==name, displayName==name +bool REMOVE_EmptyContainers = true; // empty {} and [] (preserves structural containers) + +// Presentation properties (optional) +bool REMOVE_SummarizeBy = true; // summarizeBy (all values, not just none) +bool REMOVE_DisplayProps = true; // isHidden, displayFolder +bool REMOVE_QueryGroups = false; // queryGroup, queryGroups, folder +bool REMOVE_FormatString = true; // formatString literal only (NEVER formatStringDefinition) + +// Additional metadata (recommended: ON) +bool REMOVE_ExtraMetadata = true; // sourceProviderType, isNameInferred, isDataTypeInferred + +// ==================== OUTPUT FORMAT ===================== +// Human-friendly indented JSON (false) or compacted (true) +bool MINIFY_OUTPUT = true; + +// ==================== MAIN EXECUTION ==================== +try +{ + // Select file + string inputPath; + using (var dialog = new OpenFileDialog { + Title = "Select BIM file to slim", + Filter = "Tabular Model (*.bim)|*.bim|All files (*.*)|*.*", + RestoreDirectory = true + }) { + if (dialog.ShowDialog() != DialogResult.OK) return; + inputPath = dialog.FileName; + } + + // Generate output path + var outputPath = Path.ChangeExtension(inputPath, ".slim"); + var originalSize = new FileInfo(inputPath).Length; + + // Parse JSON + var root = JToken.Parse(File.ReadAllText(inputPath)); + + // Build removal rules + var dropKeys = new HashSet(StringComparer.OrdinalIgnoreCase); + if (REMOVE_Annotations) { + dropKeys.UnionWith(new[] { "annotations", "changedProperties", "extendedProperties" }); + } + if (REMOVE_Lineage) { + dropKeys.UnionWith(new[] { "lineageTag", "sourceLineageTag" }); + } + if (REMOVE_LanguageData) { + dropKeys.UnionWith(new[] { "cultures", "translations", "synonyms", "linguisticMetadata" }); + } + if (REMOVE_SummarizeBy) { + dropKeys.Add("summarizeBy"); + } + if (REMOVE_DisplayProps) { + dropKeys.UnionWith(new[] { "isHidden", "displayFolder" }); + } + if (REMOVE_QueryGroups) { + dropKeys.UnionWith(new[] { "queryGroup", "queryGroups", "folder" }); + } + if (REMOVE_ExtraMetadata) { + dropKeys.UnionWith(new[] { "sourceProviderType", "isNameInferred", "isDataTypeInferred" }); + } + + // Structural containers - never remove even if empty (preserves model schema) + var preserve = new HashSet(new[] { + "model", "tables", "columns", "measures", "relationships", "partitions", + "roles", "hierarchies", "levels", "dataSources", "perspectives", "expressions" + }, StringComparer.OrdinalIgnoreCase); + + // Track removals + var stats = new Dictionary(); + Action Track = delegate(string key) { + stats[key] = stats.ContainsKey(key) ? stats[key] + 1 : 1; + }; + + // Helpers + Func Eq = delegate(string a, string b) { + return string.Equals( + a != null ? a.Trim() : null, + b != null ? b.Trim() : null + ); + }; + + Func IsEmpty = delegate(JToken t) { + return t == null || t.Type == JTokenType.Null || + (t is JContainer && !((JContainer)t).HasValues) || + (t.Type == JTokenType.String && string.IsNullOrWhiteSpace((string)t)); + }; + + // Recursive cleaner + Action Clean = null; + Clean = delegate(JToken token) { + if (token == null) return; + + if (token.Type == JTokenType.Object) { + var obj = (JObject)token; + + // Recurse first (depth-first) + foreach (var prop in obj.Properties().ToList()) Clean(prop.Value); + + var toRemove = new List(); + + foreach (var prop in obj.Properties()) { + // Name-based removals + if (dropKeys.Contains(prop.Name)) { + toRemove.Add(prop); + Track(prop.Name); + continue; + } + + // formatString special handling (protect formatStringDefinition) + if (REMOVE_FormatString && Eq(prop.Name, "formatString")) { + toRemove.Add(prop); + Track("formatString"); + continue; + } + + // Empty container removal (with structural preservation) + if (REMOVE_EmptyContainers && IsEmpty(prop.Value) && !preserve.Contains(prop.Name)) { + toRemove.Add(prop); + Track("empty"); + continue; + } + } + + // Value-based removals (checked after structure scan) + if (REMOVE_DefaultValues) { + var dc = obj.Property("dataCategory"); + if (dc != null && dc.Value is JValue && Eq((string)dc.Value, "Uncategorized")) { + toRemove.Add(dc); + Track("dataCategory=default"); + } + + var sb = obj.Property("summarizeBy"); + if (sb != null && sb.Value is JValue && Eq((string)sb.Value, "none")) { + toRemove.Add(sb); + Track("summarizeBy=none"); + } + } + + if (REMOVE_RedundantNames) { + var name = obj.Property("name"); + if (name != null && name.Value is JValue) { + var nameStr = (string)name.Value; + var nameBracketed = nameStr != null ? string.Format("[{0}]", nameStr) : null; + + var src = obj.Property("sourceColumn"); + if ( + src != null && + src.Value is JValue && + ( + Eq((string)src.Value, nameStr) || + Eq((string)src.Value, nameBracketed) + ) + ) { + toRemove.Add(src); + Track("sourceColumn=name"); + } + + var disp = obj.Property("displayName"); + if ( + disp != null && + disp.Value is JValue && + ( + Eq((string)disp.Value, nameStr) || + Eq((string)disp.Value, nameBracketed) + ) + ) { + toRemove.Add(disp); + Track("displayName=name"); + } + } + } + + // Apply all removals + foreach (var prop in toRemove.Distinct()) prop.Remove(); + } + else if (token.Type == JTokenType.Array) { + var arr = (JArray)token; + foreach (var item in arr.ToList()) { + Clean(item); + if (REMOVE_EmptyContainers && IsEmpty(item)) { + item.Remove(); + Track("empty"); + } + } + } + }; + + // Execute cleaning + Clean(root); + + // Save result + var formatting = MINIFY_OUTPUT ? Formatting.None : Formatting.Indented; + File.WriteAllText(outputPath, root.ToString(formatting)); + + // Report results + var newSize = new FileInfo(outputPath).Length; + var reduction = (1.0 - (double)newSize / originalSize) * 100; + var summary = + "BIM Slimmer Results\n" + + "==================\n" + + string.Format("Input: {0} ({1:N1} KB)\n", Path.GetFileName(inputPath), originalSize / 1024.0) + + string.Format("Output: {0} ({1:N1} KB)\n", Path.GetFileName(outputPath), newSize / 1024.0) + + string.Format("Saved: {0:F1}%\n\n", reduction) + + string.Format("Removed: {0:N0} items\n", stats.Values.Sum()) + + string.Join( + "\n", + stats.OrderBy(k => k.Key) + .Select(k => string.Format(" • {0}: {1:N0}", k.Key, k.Value)) + .ToArray() + ); + + Info(summary); +} +catch (Exception ex) +{ + Error(string.Format("Processing failed: {0}", ex.Message)); +} \ No newline at end of file diff --git a/Intermediate/tmdl_slimmer.csx b/Intermediate/tmdl_slimmer.csx new file mode 100644 index 0000000..ca8c3b3 --- /dev/null +++ b/Intermediate/tmdl_slimmer.csx @@ -0,0 +1,297 @@ +// +// Title: TMDL Slimmer - Strip metadata bloat for LLM context +// +// Author: Alexis Olson +// Version: 1.1 +// +// Description: +// Reads all *.tmdl files from a SemanticModel/definition folder, +// removes UI/engine metadata while preserving model semantics, +// and outputs a single .slimdl file for LLM consumption. +// +// Usage: +// - Run in Tabular Editor 2 or 3 (Advanced Scripting) +// - Select your SemanticModel folder when prompted +// - Choose where to save the output .slimdl file + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Text.RegularExpressions; +using System.Windows.Forms; + +// ==================== CONFIGURATION ==================== +bool REMOVE_Annotations = true; // annotation, changedProperty, extendedProperty/extendedProperties +bool REMOVE_Lineage = true; // lineageTag, sourceLineageTag +bool REMOVE_LanguageData = true; // cultures folder (includes linguisticMetadata) +bool REMOVE_ColumnMeta = true; // summarizeBy, sourceColumn, dataCategory (+ select column booleans) +bool REMOVE_InferredMeta = true; // isNameInferred, isDataTypeInferred, sourceProviderType +bool REMOVE_DisplayProps = true; // isHidden, displayFolder, formatString, isDefaultLabel/Image + +// ==================== MAIN EXECUTION ==================== +try +{ + // Select SemanticModel folder + string modelFolder = null; + using (var dialog = new FolderBrowserDialog()) + { + dialog.Description = "Select the SemanticModel folder (contains 'definition' subfolder)"; + dialog.ShowNewFolderButton = false; + if (dialog.ShowDialog() != DialogResult.OK) return; + modelFolder = dialog.SelectedPath; + } + + // Locate definition root - handle both cases: user selected SemanticModel or definition directly + string definitionPath = Path.Combine(modelFolder, "definition"); + if (!Directory.Exists(definitionPath)) + { + definitionPath = modelFolder; // Fallback: user already selected the definition folder + if (Directory.GetFiles(definitionPath, "*.tmdl", SearchOption.AllDirectories).Length == 0) + { + Info("No TMDL files found in the selected folder."); + return; + } + } + + // Build removal patterns based on configuration flags + var patterns = new Dictionary(); + + // Common regex components for matching property assignments + string ASSIGN = @"\s*(?:=|:)"; // matches optional whitespace then = or : + string BOOL = @"(?:\s*(?:=|:)\s*(?:true|false))?\s*;?\s*$"; // matches optional boolean and semicolon + + // Helper to add patterns when corresponding removal flag is enabled + Action Add = (flag, name, pattern) => + { + if (flag) patterns[name] = new Regex(pattern); + }; + + // Annotations group + Add(REMOVE_Annotations, "annotation", @"^\s*annotation\b"); + Add(REMOVE_Annotations, "changedProperty", @"^\s*changedProperty\b"); + Add(REMOVE_Annotations, "extendedProperty", @"^\s*extendedPropert(?:y|ies)\b"); + + // Lineage tracking group + Add(REMOVE_Lineage, "lineageTag", @"^\s*lineageTag" + ASSIGN); + Add(REMOVE_Lineage, "sourceLineageTag", @"^\s*sourceLineageTag" + ASSIGN); + + // Column metadata group + Add(REMOVE_ColumnMeta, "dataCategory", @"^\s*dataCategory" + ASSIGN); + Add(REMOVE_ColumnMeta, "summarizeBy", @"^\s*summarizeBy" + ASSIGN); + Add(REMOVE_ColumnMeta, "sourceColumn", @"^\s*sourceColumn" + ASSIGN); + Add(REMOVE_ColumnMeta, "isAvailableInMdx", @"^\s*isAvailableInMdx" + BOOL); + Add(REMOVE_ColumnMeta, "isNullable", @"^\s*isNullable" + BOOL); + + // Inferred metadata group + Add(REMOVE_InferredMeta, "isNameInferred", @"^\s*isNameInferred" + BOOL); + Add(REMOVE_InferredMeta, "isDataTypeInferred", @"^\s*isDataTypeInferred" + BOOL); + Add(REMOVE_InferredMeta, "sourceProviderType", @"^\s*sourceProviderType" + ASSIGN); + + // Display/UI properties group + Add(REMOVE_DisplayProps, "isHidden", @"^\s*isHidden" + BOOL); + Add(REMOVE_DisplayProps, "displayFolder", @"^\s*displayFolder" + ASSIGN); + Add(REMOVE_DisplayProps, "formatString", @"^\s*formatString" + ASSIGN); + Add(REMOVE_DisplayProps, "isDefaultLabel", @"^\s*isDefaultLabel" + BOOL); + Add(REMOVE_DisplayProps, "isDefaultImage", @"^\s*isDefaultImage" + BOOL); + + // Identify patterns that start multi-line blocks (need brace tracking) + var blockStarters = new HashSet(); + if (REMOVE_Annotations) { + blockStarters.Add("extendedProperty"); + } + + // Track removal statistics for summary report + var removalStats = new Dictionary(); + + // Small helper to increment removal counters deterministically + Action Bump = key => + { + int v; + if (!removalStats.TryGetValue(key, out v)) v = 0; removalStats[key] = v + 1; + }; + + // Collect all TMDL files recursively + string[] tmdlFiles = Directory.GetFiles(definitionPath, "*.tmdl", SearchOption.AllDirectories); + Array.Sort(tmdlFiles); + if (tmdlFiles.Length == 0) + { + Info("No TMDL files found in the selected folder."); + return; + } + + // Initialize output with header + var output = new StringBuilder(); + output.AppendLine("// Combined TMDL (Slim)"); + output.AppendLine("// Source: " + Path.GetFileName(modelFolder)); + output.AppendLine("// Generated: " + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); + + long originalTotalSize = 0; + long culturesBytesRemoved = 0; // bytes saved by excluding cultures/ folder + int culturesFilesSkipped = 0; // number of cultures/ tmdl files skipped + int filesWithContent = 0; + + // Calculate base path for relative file names (normalize with trailing separator) + string definitionBasePath = definitionPath.TrimEnd('\\', '/') + Path.DirectorySeparatorChar; + + // Process each TMDL file + foreach (string filePath in tmdlFiles) + { + // Calculate relative path from definition root + string relativePath = filePath.StartsWith(definitionBasePath) + ? filePath.Substring(definitionBasePath.Length) + : Path.GetFileName(filePath); + relativePath = relativePath.Replace('\\', '/'); + + // Include every file's size in input total, even if we skip its content later + long fileSize = new FileInfo(filePath).Length; + originalTotalSize += fileSize; + + // Skip entire cultures/ subtree when language data removal is enabled + if (REMOVE_LanguageData && relativePath.StartsWith("cultures/")) + { + culturesBytesRemoved += fileSize; // track savings from cultures folder + culturesFilesSkipped++; + continue; + } + + // Read file content + string content = File.ReadAllText(filePath, Encoding.UTF8); + + // Process content line by line + string[] contentLines = content.Split(new[] { "\r\n", "\n", "\r" }, StringSplitOptions.None); + + // State tracking for multi-line block removal + bool inSkippedBlock = false; + int blockBraceDepth = 0; + bool fileHasOutput = false; + + foreach (string line in contentLines) + { + // Handle multi-line block skipping (tracks nested braces) + if (inSkippedBlock) + { + blockBraceDepth += line.Split('{').Length - 1; + blockBraceDepth -= line.Split('}').Length - 1; + if (blockBraceDepth <= 0) + { + inSkippedBlock = false; + blockBraceDepth = 0; + continue; // Don't output closing brace line that ended the block + } + continue; // Continue skipping lines inside the block + } + + // Check if current line matches any removal pattern + bool shouldRemoveLine = false; + foreach (var patternEntry in patterns) + { + if (patternEntry.Value.IsMatch(line)) + { + // Check if this starts a multi-line block that needs brace tracking + if (blockStarters.Contains(patternEntry.Key)) + { + Bump(patternEntry.Key); + + // Initialize brace tracking for this block + blockBraceDepth = line.Split('{').Length - line.Split('}').Length; + inSkippedBlock = true; + shouldRemoveLine = true; + break; + } + else + { + // Single-line removal + Bump(patternEntry.Key); + shouldRemoveLine = true; + break; + } + } + } + + if (!shouldRemoveLine) + { + // Skip pure whitespace lines to reduce output bloat + if (string.IsNullOrWhiteSpace(line)) + continue; + + // Keep the line (trim trailing whitespace for consistency) + output.AppendLine(line.TrimEnd()); + fileHasOutput = true; + } + } + + if (fileHasOutput) + { + filesWithContent++; + output.AppendLine(); // Ensure separation between files + } + } + + // Squeeze excessive blank lines to maximum of one blank line + string finalOutput = Regex.Replace(output.ToString(), @"(\r?\n){3,}", Environment.NewLine + Environment.NewLine); + finalOutput = finalOutput.TrimEnd() + Environment.NewLine; // Ensure file ends with newline + + // Get output path via save dialog + var parentDir = Directory.GetParent(modelFolder); + string suggestedPath = Path.Combine(parentDir != null ? parentDir.FullName : modelFolder, + Path.GetFileName(modelFolder) + ".slimdl"); + + string outputPath; + using (var saveDialog = new SaveFileDialog()) + { + saveDialog.Title = "Save slimmed TMDL"; + saveDialog.Filter = "Slimmed TMDL (*.slimdl)|*.slimdl|TMDL files (*.tmdl)|*.tmdl|All files (*.*)|*.*"; + saveDialog.DefaultExt = "slimdl"; + saveDialog.AddExtension = true; + saveDialog.FileName = Path.GetFileName(suggestedPath); + saveDialog.InitialDirectory = Path.GetDirectoryName(suggestedPath); + saveDialog.OverwritePrompt = true; + saveDialog.CheckPathExists = true; + + if (saveDialog.ShowDialog() != DialogResult.OK) return; + outputPath = saveDialog.FileName; + } + + // Write the combined, slimmed TMDL + File.WriteAllText(outputPath, finalOutput, new UTF8Encoding(false)); + + // Calculate size reduction metrics + long outputSize = new FileInfo(outputPath).Length; + double reductionPercent = (originalTotalSize > 0) + ? (1.0 - (double)outputSize / (double)originalTotalSize) * 100.0 + : 0.0; + + // Generate summary report + var summary = new StringBuilder(); + summary.AppendLine("TMDL Slimmer Results"); + summary.AppendLine("===================="); + summary.AppendLine(string.Format("Files processed: {0} of {1}", filesWithContent, tmdlFiles.Length)); + if (culturesFilesSkipped > 0) + summary.AppendLine(string.Format("Culture files not processed: {0}", culturesFilesSkipped)); + summary.AppendLine(string.Format("Input size: {0:N1} KB", originalTotalSize / 1024.0)); + summary.AppendLine(string.Format("Output size: {0:N1} KB", outputSize / 1024.0)); + summary.AppendLine(string.Format("Size reduction: {0:F1}%", reductionPercent)); + + if (removalStats.Count > 0) + { + int totalRemovals = 0; + foreach (int count in removalStats.Values) totalRemovals += count; + summary.AppendLine(); + if (culturesBytesRemoved > 0) + summary.AppendLine(string.Format("Removed cultures folder: {0:N1} KB", culturesBytesRemoved / 1024.0)); + summary.AppendLine(); + summary.AppendLine(string.Format("Removed {0:N0} items:", totalRemovals)); + + var sortedKeys = new List(removalStats.Keys); + sortedKeys.Sort(); + foreach (string key in sortedKeys) + summary.AppendLine(string.Format(" - {0}: {1:N0}", key, removalStats[key])); + } + + Info(summary.ToString()); +} +catch (Exception ex) +{ + Error("Processing failed: " + ex.Message); +}