From c6cbdb5ccfc1dea07c03802231ff522a8080b7c4 Mon Sep 17 00:00:00 2001 From: tonghuaroot Date: Fri, 17 Jul 2026 17:58:46 +0800 Subject: [PATCH 1/3] gh-153838: Skip non-regular and oversized files in heatmap exporter --- Lib/profiling/sampling/heatmap_collector.py | 16 ++++--- Lib/test/test_profiling/test_heatmap.py | 44 +++++++++++++++++++ ...-07-17-12-00-00.gh-issue-153838.HmCp5s.rst | 2 + 3 files changed, 55 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-17-12-00-00.gh-issue-153838.HmCp5s.rst diff --git a/Lib/profiling/sampling/heatmap_collector.py b/Lib/profiling/sampling/heatmap_collector.py index 220b6b8150ac980..349ae03113b94b8 100644 --- a/Lib/profiling/sampling/heatmap_collector.py +++ b/Lib/profiling/sampling/heatmap_collector.py @@ -783,14 +783,16 @@ def _generate_file_html(self, output_path: Path, filename: str, line_counts: Dict[int, int], self_counts: Dict[int, int], file_stat: FileStats): """Generate HTML for a single source file with heatmap coloring.""" - # Read source file + _MAX_SOURCE_SIZE = 10 * 1024 * 1024 + source_lines = [f"# Source file not available: {filename}"] try: - source_lines = Path(filename).read_text(encoding='utf-8', errors='replace').splitlines() - except (IOError, OSError) as e: - if not (filename.startswith('<') or filename.startswith('[') or - filename in ('~', '...', '.') or len(filename) < 2): - print(f"Warning: Could not read source file {filename}: {e}") - source_lines = [f"# Source file not available: {filename}"] + resolved = Path(filename).resolve() + if (resolved.is_file() + and resolved.stat().st_size <= _MAX_SOURCE_SIZE): + source_lines = resolved.read_text( + encoding='utf-8', errors='replace').splitlines() + except (IOError, OSError): + pass # Generate HTML for each line max_samples = max(line_counts.values()) if line_counts else 1 diff --git a/Lib/test/test_profiling/test_heatmap.py b/Lib/test/test_profiling/test_heatmap.py index ee27fdd3fa3053c..e77745dda12fb3f 100644 --- a/Lib/test/test_profiling/test_heatmap.py +++ b/Lib/test/test_profiling/test_heatmap.py @@ -622,6 +622,50 @@ def test_export_file_html_has_line_numbers(self): # Should have line-related content self.assertIn('line-', content) + def test_export_skips_nonexistent_source(self): + """Source files that do not exist produce a placeholder.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [('/no/such/file.py', (1, 1, -1, -1), 'f', None)] + collector.process_frames(frames, thread_id=1) + + output_path = os.path.join(self.test_dir, 'missing_src') + + with captured_stdout(), captured_stderr(): + collector.export(output_path) + + html_files = [f for f in os.listdir(output_path) + if f.startswith('file_') and f.endswith('.html')] + if html_files: + with open(os.path.join(output_path, html_files[0]), + 'r', encoding='utf-8') as f: + content = f.read() + self.assertIn('Source file not available', content) + + def test_export_caps_oversized_source(self): + """Source files larger than the cap produce a placeholder.""" + collector = HeatmapCollector(sample_interval_usec=100) + + big_file = os.path.join(self.test_dir, 'big.py') + with open(big_file, 'w') as f: + f.write('x = 1\n' * 2_000_000) + + frames = [(big_file, (1, 1, -1, -1), 'f', None)] + collector.process_frames(frames, thread_id=1) + + output_path = os.path.join(self.test_dir, 'big_src') + + with captured_stdout(), captured_stderr(): + collector.export(output_path) + + html_files = [f for f in os.listdir(output_path) + if f.startswith('file_') and f.endswith('.html')] + if html_files: + with open(os.path.join(output_path, html_files[0]), + 'r', encoding='utf-8') as f: + content = f.read() + self.assertIn('Source file not available', content) + class MockFrameInfo: """Mock FrameInfo for testing. diff --git a/Misc/NEWS.d/next/Library/2026-07-17-12-00-00.gh-issue-153838.HmCp5s.rst b/Misc/NEWS.d/next/Library/2026-07-17-12-00-00.gh-issue-153838.HmCp5s.rst new file mode 100644 index 000000000000000..e55365c591cf658 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-17-12-00-00.gh-issue-153838.HmCp5s.rst @@ -0,0 +1,2 @@ +Skip non-regular and oversized files in the ``profiling.sampling`` heatmap +exporter instead of reading them unconditionally. Patch by tonghuaroot. From ef1007372f5fae07b4fe57f2a75dda7bd7b580a7 Mon Sep 17 00:00:00 2001 From: tonghuaroot Date: Fri, 17 Jul 2026 18:19:42 +0800 Subject: [PATCH 2/3] Drop the size cap, keep the is_file() guard only --- Lib/profiling/sampling/heatmap_collector.py | 4 +--- Lib/test/test_profiling/test_heatmap.py | 23 ------------------- ...-07-17-12-00-00.gh-issue-153838.HmCp5s.rst | 4 ++-- 3 files changed, 3 insertions(+), 28 deletions(-) diff --git a/Lib/profiling/sampling/heatmap_collector.py b/Lib/profiling/sampling/heatmap_collector.py index 349ae03113b94b8..ae42f38ab822443 100644 --- a/Lib/profiling/sampling/heatmap_collector.py +++ b/Lib/profiling/sampling/heatmap_collector.py @@ -783,12 +783,10 @@ def _generate_file_html(self, output_path: Path, filename: str, line_counts: Dict[int, int], self_counts: Dict[int, int], file_stat: FileStats): """Generate HTML for a single source file with heatmap coloring.""" - _MAX_SOURCE_SIZE = 10 * 1024 * 1024 source_lines = [f"# Source file not available: {filename}"] try: resolved = Path(filename).resolve() - if (resolved.is_file() - and resolved.stat().st_size <= _MAX_SOURCE_SIZE): + if resolved.is_file(): source_lines = resolved.read_text( encoding='utf-8', errors='replace').splitlines() except (IOError, OSError): diff --git a/Lib/test/test_profiling/test_heatmap.py b/Lib/test/test_profiling/test_heatmap.py index e77745dda12fb3f..961e728babf5876 100644 --- a/Lib/test/test_profiling/test_heatmap.py +++ b/Lib/test/test_profiling/test_heatmap.py @@ -642,29 +642,6 @@ def test_export_skips_nonexistent_source(self): content = f.read() self.assertIn('Source file not available', content) - def test_export_caps_oversized_source(self): - """Source files larger than the cap produce a placeholder.""" - collector = HeatmapCollector(sample_interval_usec=100) - - big_file = os.path.join(self.test_dir, 'big.py') - with open(big_file, 'w') as f: - f.write('x = 1\n' * 2_000_000) - - frames = [(big_file, (1, 1, -1, -1), 'f', None)] - collector.process_frames(frames, thread_id=1) - - output_path = os.path.join(self.test_dir, 'big_src') - - with captured_stdout(), captured_stderr(): - collector.export(output_path) - - html_files = [f for f in os.listdir(output_path) - if f.startswith('file_') and f.endswith('.html')] - if html_files: - with open(os.path.join(output_path, html_files[0]), - 'r', encoding='utf-8') as f: - content = f.read() - self.assertIn('Source file not available', content) class MockFrameInfo: diff --git a/Misc/NEWS.d/next/Library/2026-07-17-12-00-00.gh-issue-153838.HmCp5s.rst b/Misc/NEWS.d/next/Library/2026-07-17-12-00-00.gh-issue-153838.HmCp5s.rst index e55365c591cf658..faecab8cff3e7b1 100644 --- a/Misc/NEWS.d/next/Library/2026-07-17-12-00-00.gh-issue-153838.HmCp5s.rst +++ b/Misc/NEWS.d/next/Library/2026-07-17-12-00-00.gh-issue-153838.HmCp5s.rst @@ -1,2 +1,2 @@ -Skip non-regular and oversized files in the ``profiling.sampling`` heatmap -exporter instead of reading them unconditionally. Patch by tonghuaroot. +Skip non-regular files in the ``profiling.sampling`` heatmap exporter +instead of reading them unconditionally. Patch by tonghuaroot. From 1408d115717699a75fb58b0f3eed981afebc5f90 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Mon, 5 Oct 2026 15:04:29 +0100 Subject: [PATCH 3/3] gh-153838: Verify non-regular sources are never opened --- Lib/profiling/sampling/heatmap_collector.py | 6 +-- Lib/test/test_profiling/test_heatmap.py | 50 ++++++++++++--------- 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/Lib/profiling/sampling/heatmap_collector.py b/Lib/profiling/sampling/heatmap_collector.py index ae42f38ab822443..62f42755827f068 100644 --- a/Lib/profiling/sampling/heatmap_collector.py +++ b/Lib/profiling/sampling/heatmap_collector.py @@ -785,9 +785,9 @@ def _generate_file_html(self, output_path: Path, filename: str, """Generate HTML for a single source file with heatmap coloring.""" source_lines = [f"# Source file not available: {filename}"] try: - resolved = Path(filename).resolve() - if resolved.is_file(): - source_lines = resolved.read_text( + path = Path(filename) + if path.is_file(): + source_lines = path.read_text( encoding='utf-8', errors='replace').splitlines() except (IOError, OSError): pass diff --git a/Lib/test/test_profiling/test_heatmap.py b/Lib/test/test_profiling/test_heatmap.py index 961e728babf5876..1b4de0caa3d00fc 100644 --- a/Lib/test/test_profiling/test_heatmap.py +++ b/Lib/test/test_profiling/test_heatmap.py @@ -4,6 +4,7 @@ import shutil import tempfile import unittest +from unittest import mock from collections import namedtuple from pathlib import Path @@ -615,34 +616,43 @@ def test_export_file_html_has_line_numbers(self): html_files = [f for f in os.listdir(output_path) if f.startswith('file_') and f.endswith('.html')] - if html_files: - with open(os.path.join(output_path, html_files[0]), 'r', encoding='utf-8') as f: - content = f.read() + self.assertEqual(len(html_files), 1) + with open(os.path.join(output_path, html_files[0]), 'r', encoding='utf-8') as f: + content = f.read() - # Should have line-related content - self.assertIn('line-', content) + # Should have line-related content + self.assertIn('line-', content) def test_export_skips_nonexistent_source(self): - """Source files that do not exist produce a placeholder.""" - collector = HeatmapCollector(sample_interval_usec=100) - - frames = [('/no/such/file.py', (1, 1, -1, -1), 'f', None)] - collector.process_frames(frames, thread_id=1) + self.check_unavailable_source(os.path.join(self.test_dir, 'missing.py')) - output_path = os.path.join(self.test_dir, 'missing_src') + def test_export_skips_directory_source(self): + self.check_unavailable_source(self.test_dir) - with captured_stdout(), captured_stderr(): - collector.export(output_path) + @unittest.skipUnless(hasattr(os, 'mkfifo'), 'requires os.mkfifo') + def test_export_skips_fifo_source(self): + filename = os.path.join(self.test_dir, 'source.fifo') + os.mkfifo(filename) + self.check_unavailable_source(filename) - html_files = [f for f in os.listdir(output_path) - if f.startswith('file_') and f.endswith('.html')] - if html_files: - with open(os.path.join(output_path, html_files[0]), - 'r', encoding='utf-8') as f: - content = f.read() - self.assertIn('Source file not available', content) + def check_unavailable_source(self, filename): + collector = HeatmapCollector(sample_interval_usec=100) + frames = [(filename, (1, 1, -1, -1), 'f', None)] + collector.process_frames(frames, thread_id=1) + output_path = Path(self.test_dir) / 'unavailable_source' + read_text = Path.read_text + def check_read(path, *args, **kwargs): + self.assertNotEqual(path, Path(filename), + 'Non-regular source must not be opened') + return read_text(path, *args, **kwargs) + with (captured_stdout(), captured_stderr(), + mock.patch.object(Path, 'read_text', check_read)): + collector.export(output_path) + content = (output_path / collector.file_index[filename]).read_text( + encoding='utf-8') + self.assertIn('Source file not available', content) class MockFrameInfo: """Mock FrameInfo for testing.