Tutorial 03 — Arabic and Multilingual Text
On this page
Level: Intermediate
Time: ~20 minutes
Arabic Text — No GPU Required
from zerda.chunks import ArabicTextChunker
chunker = ArabicTextChunker(
chunk_size=400,
overlap=80,
use_semantic_chunking=False, # rule-based: fast, no GPU
fix_spacing=True, # fix OCR spacing issues
)
arabic_text = """
الذكاء الاصطناعي هو محاكاة العمليات الذكائية البشرية بواسطة أنظمة الحاسوب.
تشمل هذه العمليات التعلم والتفكير وإصلاح الأخطاء الذاتية.
يُستخدم الذكاء الاصطناعي في مجالات عديدة مثل الطب والهندسة والتعليم.
"""
chunks = chunker.chunk(arabic_text, doc_id="ar_001", source="ai_article_ar.txt")
for c in chunks:
print(f"[{c.char_count} chars] {c.text[:60]}")
Arabic with Normalization
Normalize before chunking to remove diacritics and Alef variants:
from zerda.chunks import (
ArabicNormalizer,
NormalizingChunkerWrapper,
ArabicTextChunker,
NormalizerConfig,
)
# Customize normalization
norm_config = NormalizerConfig(
normalize_alef=True,
remove_diacritics=True,
normalize_numbers=True,
fix_spacing=True,
)
normalizer = ArabicNormalizer(config=norm_config)
base_chunker = ArabicTextChunker(chunk_size=400, use_semantic_chunking=False)
chunker = NormalizingChunkerWrapper(base_chunker, normalizer=normalizer)
# Input with diacritics
raw = "الأُسْتاذُ يُدرِّسُ اللُّغةَ العَربيَّةَ في الجامعة"
chunks = chunker.chunk(raw, doc_id="test")
print("Normalized:", chunks[0].text)
# → "الاستاذ يدرس اللغه العربيه في الجامعه"
Detecting Arabic Ratio
chunker = ArabicTextChunker()
samples = [
"مرحباً بالعالم", # pure Arabic
"Hello مرحبا world", # mixed
"This is English only", # no Arabic
]
for s in samples:
ratio = chunker.get_arabic_ratio(s)
dominant = chunker.is_arabic_dominant(s)
print(f"ratio={ratio:.0%} dominant={dominant} text={s[:30]}")
Multilingual — Auto-Detect
from zerda.chunks import MultilanguageTextChunker
chunker = MultilanguageTextChunker(chunk_size=512, language="auto")
texts = {
"arabic": "مرحباً بالعالم. هذا نص عربي طويل.",
"chinese": "人工智能是计算机科学的一个分支，研究如何使计算机完成需要人类智能的工作。",
"english": "Artificial intelligence is transforming how we interact with technology.",
"french": "L'intelligence artificielle transforme notre façon d'interagir avec la technologie.",
"russian": "Искусственный интеллект меняет то, как мы взаимодействуем с технологиями.",
}
for lang, text in texts.items():
chunks = chunker.chunk(text, source=f"{lang}.txt")
detected = chunks[0].metadata.language if chunks else "n/a"
print(f"Input: {lang:10s} Detected: {detected}")
Processing a Mixed-Language Corpus
from zerda.chunks import MultilanguageTextChunker, Document
chunker = MultilanguageTextChunker(chunk_size=400, overlap=80)
docs = [
Document(page_content=ar_text, metadata={"source": "ar_doc.txt"}),
Document(page_content=en_text, metadata={"source": "en_doc.txt"}),
Document(page_content=zh_text, metadata={"source": "zh_doc.txt"}),
]
all_chunks = chunker.chunk_documents(docs)
# Group by language
from collections import Counter
lang_counts = Counter(c.metadata.language for c in all_chunks)
print(lang_counts)
# Counter({'arabic': 5, 'english': 3, 'chinese': 2})