vibe-coding-cn/libs/external/Skill_Seekers-development/tests/test_github_scraper.py

#!/usr/bin/env python3
"""
Tests for GitHub Scraper (cli/github_scraper.py)

Tests cover:
- GitHubScraper initialization and configuration (C1.1)
- README extraction (C1.2)
- Language detection (C1.4)
- GitHub Issues extraction (C1.7)
- CHANGELOG extraction (C1.8)
- GitHub Releases extraction (C1.9)
- GitHubToSkillConverter and skill building (C1.10)
- Authentication handling
- Error handling and edge cases
"""

import unittest
import sys
import json
import tempfile
import shutil
import os
from pathlib import Path
from unittest.mock import Mock, patch, MagicMock
from datetime import datetime

try:
    from github import Github, GithubException
    PYGITHUB_AVAILABLE = True
except ImportError:
    PYGITHUB_AVAILABLE = False


class TestGitHubScraperInitialization(unittest.TestCase):
    """Test GitHubScraper initialization and configuration (C1.1)"""

    def setUp(self):
        if not PYGITHUB_AVAILABLE:
            self.skipTest("PyGithub not installed")
        from skill_seekers.cli.github_scraper import GitHubScraper
        self.GitHubScraper = GitHubScraper

        # Create temporary directory for test output
        self.temp_dir = tempfile.mkdtemp()
        self.output_dir = Path(self.temp_dir)

    def tearDown(self):
        # Clean up temporary directory
        if hasattr(self, 'temp_dir'):
            shutil.rmtree(self.temp_dir, ignore_errors=True)

    def test_init_with_repo_name(self):
        """Test initialization with repository name"""
        config = {
            'repo': 'facebook/react',
            'name': 'react',
            'github_token': None
        }

        scraper = self.GitHubScraper(config)

        self.assertEqual(scraper.repo_name, 'facebook/react')
        self.assertEqual(scraper.name, 'react')
        self.assertIsNotNone(scraper.github)

    def test_init_with_token_from_config(self):
        """Test initialization with token from config"""
        config = {
            'repo': 'facebook/react',
            'name': 'react',
            'github_token': 'test_token_123'
        }

        with patch('skill_seekers.cli.github_scraper.Github') as mock_github:
            scraper = self.GitHubScraper(config)
            mock_github.assert_called_once_with('test_token_123')

    def test_init_with_token_from_env(self):
        """Test initialization with token from environment variable"""
        config = {
            'repo': 'facebook/react',
            'name': 'react',
            'github_token': None
        }

        with patch.dict(os.environ, {'GITHUB_TOKEN': 'env_token_456'}):
            with patch('skill_seekers.cli.github_scraper.Github') as mock_github:
                scraper = self.GitHubScraper(config)
                mock_github.assert_called_once_with('env_token_456')

    def test_init_without_token(self):
        """Test initialization without authentication"""
        config = {
            'repo': 'facebook/react',
            'name': 'react',
            'github_token': None
        }

        with patch('skill_seekers.cli.github_scraper.Github') as mock_github:
            with patch.dict(os.environ, {}, clear=True):
                scraper = self.GitHubScraper(config)
                # Should create unauthenticated client
                self.assertIsNotNone(scraper.github)

    def test_token_priority_env_over_config(self):
        """Test that GITHUB_TOKEN env var takes priority over config"""
        config = {
            'repo': 'facebook/react',
            'name': 'react',
            'github_token': 'config_token'
        }

        with patch.dict(os.environ, {'GITHUB_TOKEN': 'env_token'}):
            scraper = self.GitHubScraper(config)
            token = scraper._get_token()
            self.assertEqual(token, 'env_token')


class TestREADMEExtraction(unittest.TestCase):
    """Test README extraction (C1.2)"""

    def setUp(self):
        if not PYGITHUB_AVAILABLE:
            self.skipTest("PyGithub not installed")
        from skill_seekers.cli.github_scraper import GitHubScraper
        self.GitHubScraper = GitHubScraper

    def test_extract_readme_success(self):
        """Test successful README extraction"""
        config = {
            'repo': 'facebook/react',
            'name': 'react',
            'github_token': None
        }

        mock_content = Mock()
        mock_content.decoded_content = b'# React\n\nA JavaScript library'

        with patch('skill_seekers.cli.github_scraper.Github'):
            scraper = self.GitHubScraper(config)
            scraper.repo = Mock()
            scraper.repo.get_contents.return_value = mock_content

            scraper._extract_readme()

            self.assertIn('readme', scraper.extracted_data)
            self.assertEqual(scraper.extracted_data['readme'], '# React\n\nA JavaScript library')

    def test_extract_readme_tries_multiple_locations(self):
        """Test that README extraction tries multiple file locations"""
        config = {
            'repo': 'facebook/react',
            'name': 'react',
            'github_token': None
        }

        with patch('skill_seekers.cli.github_scraper.Github'):
            scraper = self.GitHubScraper(config)
            scraper.repo = Mock()

            # Make first attempts fail, succeed on third
            def side_effect(path):
                if path in ['README.md', 'README.rst']:
                    raise GithubException(404, 'Not found')
                mock_content = Mock()
                mock_content.decoded_content = b'# README'
                return mock_content

            scraper.repo.get_contents.side_effect = side_effect

            scraper._extract_readme()

            # Should have tried multiple paths
            self.assertGreaterEqual(scraper.repo.get_contents.call_count, 1)

    def test_extract_readme_not_found(self):
        """Test README extraction when no README exists"""
        config = {
            'repo': 'test/norepo',
            'name': 'norepo',
            'github_token': None
        }

        with patch('skill_seekers.cli.github_scraper.Github'):
            scraper = self.GitHubScraper(config)
            scraper.repo = Mock()
            scraper.repo.get_contents.side_effect = GithubException(404, 'Not found')

            scraper._extract_readme()

            # Should not crash, just log warning (readme initialized as empty string)
            self.assertEqual(scraper.extracted_data['readme'], '')


class TestLanguageDetection(unittest.TestCase):
    """Test language detection (C1.4)"""

    def setUp(self):
        if not PYGITHUB_AVAILABLE:
            self.skipTest("PyGithub not installed")
        from skill_seekers.cli.github_scraper import GitHubScraper
        self.GitHubScraper = GitHubScraper

    def test_extract_languages_success(self):
        """Test successful language detection"""
        config = {
            'repo': 'facebook/react',
            'name': 'react',
            'github_token': None
        }

        with patch('skill_seekers.cli.github_scraper.Github'):
            scraper = self.GitHubScraper(config)
            scraper.repo = Mock()
            scraper.repo.get_languages.return_value = {
                'JavaScript': 8000,
                'TypeScript': 2000
            }

            scraper._extract_languages()

            self.assertIn('languages', scraper.extracted_data)
            self.assertIn('JavaScript', scraper.extracted_data['languages'])
            self.assertIn('TypeScript', scraper.extracted_data['languages'])

            # Check percentages
            js_data = scraper.extracted_data['languages']['JavaScript']
            self.assertEqual(js_data['bytes'], 8000)
            self.assertEqual(js_data['percentage'], 80.0)

            ts_data = scraper.extracted_data['languages']['TypeScript']
            self.assertEqual(ts_data['bytes'], 2000)
            self.assertEqual(ts_data['percentage'], 20.0)

    def test_extract_languages_empty(self):
        """Test language detection with no languages"""
        config = {
            'repo': 'test/norepo',
            'name': 'norepo',
            'github_token': None
        }

        with patch('skill_seekers.cli.github_scraper.Github'):
            scraper = self.GitHubScraper(config)
            scraper.repo = Mock()
            scraper.repo.get_languages.return_value = {}

            scraper._extract_languages()

            self.assertIn('languages', scraper.extracted_data)
            self.assertEqual(scraper.extracted_data['languages'], {})


class TestIssuesExtraction(unittest.TestCase):
    """Test GitHub Issues extraction (C1.7)"""

    def setUp(self):
        if not PYGITHUB_AVAILABLE:
            self.skipTest("PyGithub not installed")
        from skill_seekers.cli.github_scraper import GitHubScraper
        self.GitHubScraper = GitHubScraper

    def test_extract_issues_success(self):
        """Test successful issues extraction"""
        config = {
            'repo': 'facebook/react',
            'name': 'react',
            'github_token': None,
            'max_issues': 10
        }

        # Create mock issues
        mock_label1 = Mock()
        mock_label1.name = 'bug'
        mock_label2 = Mock()
        mock_label2.name = 'high-priority'

        mock_milestone = Mock()
        mock_milestone.title = 'v18.0'

        mock_issue1 = Mock()
        mock_issue1.number = 123
        mock_issue1.title = 'Bug in useState'
        mock_issue1.state = 'open'
        mock_issue1.labels = [mock_label1, mock_label2]
        mock_issue1.milestone = mock_milestone
        mock_issue1.created_at = datetime(2023, 1, 1)
        mock_issue1.updated_at = datetime(2023, 1, 2)
        mock_issue1.closed_at = None
        mock_issue1.html_url = 'https://github.com/facebook/react/issues/123'
        mock_issue1.body = 'Issue description'
        mock_issue1.pull_request = None

        mock_label3 = Mock()
        mock_label3.name = 'enhancement'

        mock_issue2 = Mock()
        mock_issue2.number = 124
        mock_issue2.title = 'Feature request'
        mock_issue2.state = 'closed'
        mock_issue2.labels = [mock_label3]
        mock_issue2.milestone = None
        mock_issue2.created_at = datetime(2023, 1, 3)
        mock_issue2.updated_at = datetime(2023, 1, 4)
        mock_issue2.closed_at = datetime(2023, 1, 5)
        mock_issue2.html_url = 'https://github.com/facebook/react/issues/124'
        mock_issue2.body = 'Feature description'
        mock_issue2.pull_request = None

        with patch('skill_seekers.cli.github_scraper.Github'):
            scraper = self.GitHubScraper(config)
            scraper.repo = Mock()
            scraper.repo.get_issues.return_value = [mock_issue1, mock_issue2]

            scraper._extract_issues()

            self.assertIn('issues', scraper.extracted_data)
            issues = scraper.extracted_data['issues']
            self.assertEqual(len(issues), 2)

            # Check first issue
            self.assertEqual(issues[0]['number'], 123)
            self.assertEqual(issues[0]['title'], 'Bug in useState')
            self.assertEqual(issues[0]['state'], 'open')
            self.assertEqual(issues[0]['labels'], ['bug', 'high-priority'])
            self.assertEqual(issues[0]['milestone'], 'v18.0')

            # Check second issue
            self.assertEqual(issues[1]['number'], 124)
            self.assertEqual(issues[1]['state'], 'closed')
            self.assertIsNone(issues[1]['milestone'])

    def test_extract_issues_filters_pull_requests(self):
        """Test that pull requests are filtered out from issues"""
        config = {
            'repo': 'facebook/react',
            'name': 'react',
            'github_token': None,
            'max_issues': 10
        }

        # Create mock issue (need all required attributes)
        mock_issue = Mock()
        mock_issue.number = 123
        mock_issue.title = 'Real issue'
        mock_issue.state = 'open'
        mock_issue.labels = []
        mock_issue.milestone = None
        mock_issue.created_at = datetime(2023, 1, 1)
        mock_issue.updated_at = datetime(2023, 1, 2)
        mock_issue.closed_at = None
        mock_issue.html_url = 'https://github.com/test/repo/issues/123'
        mock_issue.body = 'Issue body'
        mock_issue.pull_request = None

        mock_pr = Mock()
        mock_pr.number = 124
        mock_pr.title = 'Pull request'
        mock_pr.pull_request = Mock()  # Has pull_request attribute

        with patch('skill_seekers.cli.github_scraper.Github'):
            scraper = self.GitHubScraper(config)
            scraper.repo = Mock()
            scraper.repo.get_issues.return_value = [mock_issue, mock_pr]

            scraper._extract_issues()

            issues = scraper.extracted_data['issues']
            # Should only have the real issue, not the PR
            self.assertEqual(len(issues), 1)
            self.assertEqual(issues[0]['number'], 123)

    def test_extract_issues_respects_max_limit(self):
        """Test that max_issues limit is respected"""
        config = {
            'repo': 'facebook/react',
            'name': 'react',
            'github_token': None,
            'max_issues': 2
        }

        # Create 5 mock issues
        mock_issues = []
        for i in range(5):
            mock_issue = Mock()
            mock_issue.number = i
            mock_issue.title = f'Issue {i}'
            mock_issue.state = 'open'
            mock_issue.labels = []
            mock_issue.milestone = None
            mock_issue.created_at = datetime(2023, 1, 1)
            mock_issue.updated_at = datetime(2023, 1, 2)
            mock_issue.closed_at = None
            mock_issue.html_url = f'https://github.com/test/repo/issues/{i}'
            mock_issue.body = None
            mock_issue.pull_request = None
            mock_issues.append(mock_issue)

        with patch('skill_seekers.cli.github_scraper.Github'):
            scraper = self.GitHubScraper(config)
            scraper.repo = Mock()
            scraper.repo.get_issues.return_value = mock_issues

            scraper._extract_issues()

            issues = scraper.extracted_data['issues']
            # Should only extract first 2 issues
            self.assertEqual(len(issues), 2)


class TestChangelogExtraction(unittest.TestCase):
    """Test CHANGELOG extraction (C1.8)"""

    def setUp(self):
        if not PYGITHUB_AVAILABLE:
            self.skipTest("PyGithub not installed")
        from skill_seekers.cli.github_scraper import GitHubScraper
        self.GitHubScraper = GitHubScraper

    def test_extract_changelog_success(self):
        """Test successful CHANGELOG extraction"""
        config = {
            'repo': 'facebook/react',
            'name': 'react',
            'github_token': None
        }

        mock_content = Mock()
        mock_content.decoded_content = b'# Changelog\n\n## v1.0.0\n- Initial release'

        with patch('skill_seekers.cli.github_scraper.Github'):
            scraper = self.GitHubScraper(config)
            scraper.repo = Mock()
            scraper.repo.get_contents.return_value = mock_content

            scraper._extract_changelog()

            self.assertIn('changelog', scraper.extracted_data)
            self.assertIn('Initial release', scraper.extracted_data['changelog'])

    def test_extract_changelog_tries_multiple_locations(self):
        """Test that CHANGELOG extraction tries multiple file locations"""
        config = {
            'repo': 'facebook/react',
            'name': 'react',
            'github_token': None
        }

        with patch('skill_seekers.cli.github_scraper.Github'):
            scraper = self.GitHubScraper(config)
            scraper.repo = Mock()

            # Make first attempts fail
            call_count = {'count': 0}

            def side_effect(path):
                call_count['count'] += 1
                if path in ['CHANGELOG.md', 'CHANGES.md']:
                    raise GithubException(404, 'Not found')
                mock_content = Mock()
                mock_content.decoded_content = b'# History'
                return mock_content

            scraper.repo.get_contents.side_effect = side_effect

            scraper._extract_changelog()

            # Should have tried multiple paths
            self.assertGreaterEqual(call_count['count'], 1)

    def test_extract_changelog_not_found(self):
        """Test CHANGELOG extraction when no changelog exists"""
        config = {
            'repo': 'test/norepo',
            'name': 'norepo',
            'github_token': None
        }

        with patch('skill_seekers.cli.github_scraper.Github'):
            scraper = self.GitHubScraper(config)
            scraper.repo = Mock()
            scraper.repo.get_contents.side_effect = GithubException(404, 'Not found')

            scraper._extract_changelog()

            # Should not crash, just log warning (changelog initialized as empty string)
            self.assertEqual(scraper.extracted_data['changelog'], '')


class TestReleasesExtraction(unittest.TestCase):
    """Test GitHub Releases extraction (C1.9)"""

    def setUp(self):
        if not PYGITHUB_AVAILABLE:
            self.skipTest("PyGithub not installed")
        from skill_seekers.cli.github_scraper import GitHubScraper
        self.GitHubScraper = GitHubScraper

    def test_extract_releases_success(self):
        """Test successful releases extraction"""
        config = {
            'repo': 'facebook/react',
            'name': 'react',
            'github_token': None
        }

        # Create mock releases
        mock_release1 = Mock()
        mock_release1.tag_name = 'v18.0.0'
        mock_release1.title = 'React 18.0.0'
        mock_release1.body = 'New features:\n- Concurrent rendering'
        mock_release1.draft = False
        mock_release1.prerelease = False
        mock_release1.created_at = datetime(2023, 3, 1)
        mock_release1.published_at = datetime(2023, 3, 1)
        mock_release1.html_url = 'https://github.com/facebook/react/releases/tag/v18.0.0'
        mock_release1.tarball_url = 'https://github.com/facebook/react/archive/v18.0.0.tar.gz'
        mock_release1.zipball_url = 'https://github.com/facebook/react/archive/v18.0.0.zip'

        mock_release2 = Mock()
        mock_release2.tag_name = 'v18.0.0-rc.0'
        mock_release2.title = 'React 18.0.0 RC'
        mock_release2.body = 'Release candidate'
        mock_release2.draft = False
        mock_release2.prerelease = True
        mock_release2.created_at = datetime(2023, 2, 1)
        mock_release2.published_at = datetime(2023, 2, 1)
        mock_release2.html_url = 'https://github.com/facebook/react/releases/tag/v18.0.0-rc.0'
        mock_release2.tarball_url = 'https://github.com/facebook/react/archive/v18.0.0-rc.0.tar.gz'
        mock_release2.zipball_url = 'https://github.com/facebook/react/archive/v18.0.0-rc.0.zip'

        with patch('skill_seekers.cli.github_scraper.Github'):
            scraper = self.GitHubScraper(config)
            scraper.repo = Mock()
            scraper.repo.get_releases.return_value = [mock_release1, mock_release2]

            scraper._extract_releases()

            self.assertIn('releases', scraper.extracted_data)
            releases = scraper.extracted_data['releases']
            self.assertEqual(len(releases), 2)

            # Check first release
            self.assertEqual(releases[0]['tag_name'], 'v18.0.0')
            self.assertEqual(releases[0]['name'], 'React 18.0.0')
            self.assertFalse(releases[0]['draft'])
            self.assertFalse(releases[0]['prerelease'])
            self.assertIn('Concurrent rendering', releases[0]['body'])

            # Check second release (prerelease)
            self.assertEqual(releases[1]['tag_name'], 'v18.0.0-rc.0')
            self.assertTrue(releases[1]['prerelease'])

    def test_extract_releases_empty(self):
        """Test releases extraction with no releases"""
        config = {
            'repo': 'test/norepo',
            'name': 'norepo',
            'github_token': None
        }

        with patch('skill_seekers.cli.github_scraper.Github'):
            scraper = self.GitHubScraper(config)
            scraper.repo = Mock()
            scraper.repo.get_releases.return_value = []

            scraper._extract_releases()

            self.assertIn('releases', scraper.extracted_data)
            self.assertEqual(scraper.extracted_data['releases'], [])


class TestGitHubToSkillConverter(unittest.TestCase):
    """Test GitHubToSkillConverter and skill building (C1.10)"""

    def setUp(self):
        if not PYGITHUB_AVAILABLE:
            self.skipTest("PyGithub not installed")
        from skill_seekers.cli.github_scraper import GitHubToSkillConverter
        self.GitHubToSkillConverter = GitHubToSkillConverter

        # Create temporary directory for test output
        self.temp_dir = tempfile.mkdtemp()
        self.output_dir = Path(self.temp_dir)

        # Create mock data file
        self.data_file = self.output_dir / "test_github_data.json"
        self.mock_data = {
            'repo_info': {
                'name': 'react',
                'full_name': 'facebook/react',
                'description': 'A JavaScript library',
                'stars': 200000,
                'language': 'JavaScript'
            },
            'readme': '# React\n\nA JavaScript library for building user interfaces.',
            'languages': {
                'JavaScript': {'bytes': 8000, 'percentage': 80.0},
                'TypeScript': {'bytes': 2000, 'percentage': 20.0}
            },
            'issues': [
                {
                    'number': 123,
                    'title': 'Bug in useState',
                    'state': 'open',
                    'labels': ['bug'],
                    'milestone': 'v18.0',
                    'created_at': '2023-01-01T10:00:00',
                    'updated_at': '2023-01-02T10:00:00',
                    'closed_at': None,
                    'url': 'https://github.com/facebook/react/issues/123',
                    'body': 'Issue description'
                }
            ],
            'changelog': '# Changelog\n\n## v18.0.0\n- New features',
            'releases': [
                {
                    'tag_name': 'v18.0.0',
                    'name': 'React 18.0.0',
                    'body': 'Release notes',
                    'published_at': '2023-03-01T10:00:00',
                    'prerelease': False,
                    'draft': False,
                    'url': 'https://github.com/facebook/react/releases/tag/v18.0.0'
                }
            ]
        }

        with open(self.data_file, 'w') as f:
            json.dump(self.mock_data, f)

    def tearDown(self):
        # Clean up temporary directory
        if hasattr(self, 'temp_dir'):
            shutil.rmtree(self.temp_dir, ignore_errors=True)

    def test_init_loads_data(self):
        """Test that converter loads data file on initialization"""
        config = {
            'repo': 'facebook/react',
            'name': 'test',
            'description': 'Test skill'
        }

        # Override data file path
        with patch('skill_seekers.cli.github_scraper.GitHubToSkillConverter.__init__') as mock_init:
            mock_init.return_value = None
            converter = self.GitHubToSkillConverter(config)
            converter.data_file = str(self.data_file)
            converter.data = converter._load_data()

            self.assertIn('repo_info', converter.data)
            self.assertEqual(converter.data['repo_info']['name'], 'react')

    def test_build_skill_creates_directory_structure(self):
        """Test that build_skill creates proper directory structure"""
        # Create data file in expected location
        data_file_path = self.output_dir / 'test_github_data.json'
        with open(data_file_path, 'w') as f:
            json.dump(self.mock_data, f)

        config = {
            'repo': 'facebook/react',
            'name': 'test',
            'description': 'Test skill'
        }

        # Patch the paths to use our temp directory
        with patch('skill_seekers.cli.github_scraper.GitHubToSkillConverter._load_data') as mock_load:
            mock_load.return_value = self.mock_data
            converter = self.GitHubToSkillConverter(config)
            converter.skill_dir = str(self.output_dir / 'test_skill')
            converter.data = self.mock_data

            converter.build_skill()

            skill_dir = Path(converter.skill_dir)
            self.assertTrue(skill_dir.exists())
            self.assertTrue((skill_dir / 'SKILL.md').exists())
            self.assertTrue((skill_dir / 'references').exists())


class TestErrorHandling(unittest.TestCase):
    """Test error handling and edge cases"""

    def setUp(self):
        if not PYGITHUB_AVAILABLE:
            self.skipTest("PyGithub not installed")
        from skill_seekers.cli.github_scraper import GitHubScraper
        self.GitHubScraper = GitHubScraper

    def test_invalid_repo_name(self):
        """Test handling of invalid repository name"""
        config = {
            'repo': 'invalid_repo_format',
            'name': 'test',
            'github_token': None
        }

        with patch('skill_seekers.cli.github_scraper.Github'):
            scraper = self.GitHubScraper(config)
            scraper.repo = None
            scraper.github.get_repo = Mock(side_effect=GithubException(404, 'Not found'))

            # Should raise ValueError with helpful message
            with self.assertRaises(ValueError) as context:
                scraper._fetch_repository()

            self.assertIn('Repository not found', str(context.exception))

    def test_rate_limit_error(self):
        """Test handling of rate limit errors"""
        config = {
            'repo': 'facebook/react',
            'name': 'react',
            'github_token': None,
            'max_issues': 10
        }

        with patch('skill_seekers.cli.github_scraper.Github'):
            scraper = self.GitHubScraper(config)
            scraper.repo = Mock()
            scraper.repo.get_issues.side_effect = GithubException(403, 'Rate limit exceeded')

            # Should handle gracefully and log warning
            scraper._extract_issues()
            # Should not crash, just log warning


if __name__ == '__main__':
    unittest.main()