| # lxml User Guide — PythonSTB Android App | |
| Generated by RIMI | |
| ## What is lxml? | |
| lxml is the most feature-rich and performance-optimized library for processing XML | |
| and HTML in Python. It wraps the C libraries libxml2 and libxslt, providing a | |
| Pythonic API while maintaining near-native speed. | |
| --- | |
| ## Why We Need It | |
| PythonSTB uses lxml for: | |
| 1. **M3U playlist parsing** — Parse HLS/DASH playlists (XML-based) | |
| 2. **EPG (Electronic Program Guide)** — Parse XMLTV format EPG data | |
| 3. **HTML scraping** — Extract data from web pages (stream URLs, titles) | |
| 4. **SOAP/XML-RPC communication** — Some IPTV APIs use XML responses | |
| 5. **XSLT transforms** — Convert between XML formats | |
| --- | |
| ## Installed Packages | |
| | Package | Version | Description | | |
| |---------|---------|-------------| | |
| | lxml | 6.1.1 | Core XML/HTML processing | | |
| | lxml_html_clean | 0.4.5 | HTML sanitizer (bundled) | | |
| --- | |
| ## Basic Usage | |
| ### Parsing XML | |
| ```python | |
| from lxml import etree | |
| # Parse from string | |
| xml_string = '''<?xml version="1.0" encoding="UTF-8"?> | |
| <channel> | |
| <name>Channel 1</name> | |
| <url>http://example.com/stream.m3u8</url> | |
| <logo>http://example.com/logo.png</logo> | |
| </channel>''' | |
| root = etree.fromstring(xml_string.encode()) | |
| print(root.find('name').text) # Channel 1 | |
| print(root.find('url').text) # http://example.com/stream.m3u8 | |
| # Parse from file | |
| tree = etree.parse('playlist.xml') | |
| root = tree.getroot() | |
| ``` | |
| ### Parsing HTML | |
| ```python | |
| from lxml import html | |
| # Parse HTML string | |
| page = html.fromstring(''' | |
| <html> | |
| <body> | |
| <div class="stream"> | |
| <h2>Channel Name</h2> | |
| <a href="http://example.com/stream.m3u8">Watch</a> | |
| </div> | |
| </body> | |
| </html>''') | |
| # XPath to find elements | |
| channels = page.xpath('//div[@class="stream"]') | |
| for ch in channels: | |
| name = ch.xpath('.//h2/text()')[0] | |
| url = ch.xpath('.//a/@href')[0] | |
| print(f"{name}: {url}") | |
| ``` | |
| ### M3U Playlist Parsing | |
| ```python | |
| from lxml import etree | |
| def parse_m3u(content): | |
| """Parse M3U/M3U8 playlist into list of channels.""" | |
| channels = [] | |
| lines = content.strip().split('\n') | |
| i = 0 | |
| while i < len(lines): | |
| line = lines[i].strip() | |
| if line.startswith('#EXTINF:'): | |
| # Parse info line | |
| info = line[8:] # Remove #EXTINF: | |
| attrs = {} | |
| # Extract duration | |
| if ',' in info: | |
| duration, name = info.rsplit(',', 1) | |
| attrs['duration'] = duration | |
| attrs['name'] = name.strip() | |
| # Next line should be the URL | |
| if i + 1 < len(lines): | |
| url = lines[i + 1].strip() | |
| if not url.startswith('#'): | |
| attrs['url'] = url | |
| channels.append(attrs) | |
| i += 2 | |
| continue | |
| i += 1 | |
| return channels | |
| # Usage | |
| with open('playlist.m3u', 'r', encoding='utf-8') as f: | |
| content = f.read() | |
| channels = parse_m3u(content) | |
| for ch in channels: | |
| print(f"{ch.get('name', 'Unknown')}: {ch.get('url', 'N/A')}") | |
| ``` | |
| ### EPG (XMLTV) Parsing | |
| ```python | |
| from lxml import etree | |
| from datetime import datetime | |
| def parse_epg(xml_content): | |
| """Parse XMLTV format EPG data.""" | |
| root = etree.fromstring(xml_content.encode()) | |
| channels = {} | |
| for channel in root.findall('.//channel'): | |
| ch_id = channel.get('id') | |
| display_name = channel.find('display-name').text | |
| icon = channel.find('icon') | |
| icon_url = icon.get('src') if icon is not None else None | |
| channels[ch_id] = { | |
| 'name': display_name, | |
| 'icon': icon_url | |
| } | |
| programmes = [] | |
| for prog in root.findall('.//programme'): | |
| programmes.append({ | |
| 'channel': prog.get('channel'), | |
| 'start': prog.get('start'), | |
| 'stop': prog.get('stop'), | |
| 'title': prog.find('title').text if prog.find('title') is not None else '', | |
| 'desc': prog.find('desc').text if prog.find('desc') is not None else '' | |
| }) | |
| return channels, programmes | |
| ``` | |
| ### XPath Examples | |
| ```python | |
| from lxml import etree | |
| tree = etree.parse('data.xml') | |
| root = tree.getroot() | |
| # Find all elements with attribute | |
| elements = root.xpath('//item[@type="channel"]') | |
| # Find with text content | |
| items = root.xpath('//item[name="Channel 1"]') | |
| # Find with contains | |
| links = root.xpath('//a[contains(@href, "m3u8")]') | |
| # Find with multiple conditions | |
| results = root.xpath('//channel[@lang="en" and @country="US"]') | |
| # Find parent element | |
| child = root.find('.//child') | |
| parent = child.getparent() | |
| # Find siblings | |
| next_sibling = child.getnext() | |
| prev_sibling = child.getprevious() | |
| ``` | |
| --- | |
| ## Features Used in App | |
| | Feature | Module | Usage | | |
| |---------|--------|-------| | |
| | XML parsing | etree | EPG, API responses | | |
| | HTML parsing | html | Web scraping | | |
| | XPath | etree | Data extraction | | |
| | ElementTree API | etree | Tree manipulation | | |
| | HTML cleaning | lxml_html_clean | Sanitize scraped HTML | | |
| | XSLT | etree | Format conversion | | |
| | C14N | etree | Canonical XML output | | |
| --- | |
| ## Troubleshooting | |
| ### Import Error | |
| ```python | |
| # If lxml fails to import: | |
| try: | |
| from lxml import etree | |
| print("lxml is installed correctly") | |
| except ImportError as e: | |
| print(f"lxml import error: {e}") | |
| # Check if .so files exist and have correct permissions | |
| ``` | |
| ### Encoding Issues | |
| ```python | |
| # Always specify encoding when parsing | |
| tree = etree.parse('file.xml') # Auto-detect | |
| tree = etree.parse(open('file.xml', 'rb')) # Binary mode | |
| # Force encoding | |
| content = open('file.xml', 'r', encoding='latin-1').read() | |
| root = etree.fromstring(content.encode('utf-8')) | |
| ``` | |
| ### Memory Errors | |
| ```python | |
| # Use iterparse for large files | |
| import xml.etree.ElementTree as ET # fallback for huge files | |
| # Or use lxml's iterparse | |
| context = etree.iterparse('large.xml', events=('end',), tag='item') | |
| for event, elem in context: | |
| process(elem) | |
| elem.clear() # Free memory | |
| ``` | |
| --- | |
| ## Related Packages | |
| | Package | Purpose | | |
| |---------|---------| | |
| | lxml_html_clean | HTML sanitization | | |
| | cssselect | CSS selector support | | |
| | cssutils | CSS parsing | | |
| | beautifulsoup4 | Alternative HTML parser | | |
| --- | |
| ## Android-Specific Notes | |
| - lxml is compiled with `-Wl,-z,norelro` for Android compatibility | |
| - Uses 16KB page alignment for Android 15+ support | |
| - Links statically against libxml2 and libxslt (no external .so files) | |
| - All `.so` files include `librimi.so` dependency | |
| Generated by RIMI | |
| --- | |
| ## App Integration | |
| ```python | |
| # Example: Full M3U parser with EPG support | |
| from lxml import etree | |
| from lxml.html import fromstring | |
| class IPTVParser: | |
| def __init__(self): | |
| self.channels = [] | |
| def parse_m3u(self, url): | |
| """Fetch and parse M3U playlist.""" | |
| import urllib.request | |
| with urllib.request.urlopen(url) as response: | |
| content = response.read().decode('utf-8') | |
| return self._parse_m3u_content(content) | |
| def _parse_m3u_content(self, content): | |
| """Parse M3U content string.""" | |
| channels = [] | |
| lines = content.strip().split('\n') | |
| for i, line in enumerate(lines): | |
| if line.startswith('#EXTINF:'): | |
| # Parse attributes | |
| attrs = {} | |
| if ',' in line: | |
| attrs['name'] = line.split(',')[-1].strip() | |
| # Get URL (next non-comment line) | |
| url = lines[i + 1].strip() if i + 1 < len(lines) else '' | |
| if url and not url.startswith('#'): | |
| attrs['url'] = url | |
| channels.append(attrs) | |
| return channels | |
| def parse_epg(self, xml_content): | |
| """Parse XMLTV EPG data.""" | |
| root = etree.fromstring(xml_content.encode()) | |
| return { | |
| 'channels': self._parse_epg_channels(root), | |
| 'programmes': self._parse_epg_programmes(root) | |
| } | |
| def _parse_epg_channels(self, root): | |
| channels = {} | |
| for ch in root.findall('.//channel'): | |
| channels[ch.get('id')] = ch.find('display-name').text | |
| return channels | |
| def _parse_epg_programmes(self, root): | |
| programmes = [] | |
| for prog in root.findall('.//programme'): | |
| programmes.append({ | |
| 'channel': prog.get('channel'), | |
| 'title': prog.find('title').text, | |
| 'start': prog.get('start') | |
| }) | |
| return programmes | |
| ``` | |