Parse interlinear to csv: Difference between revisions

    with open(input_file, mode='r', encoding='utf-8') as infile:

        reader = csv.DictReader(infile)

        # Prepare the fieldnames for the output CSV

        fieldnames = ['id', 'book', 'chapter', 'verse', 'word', 'text']

        # Open the output file to write the transformed data

        with open(output_file, mode='w', encoding='utf-8', newline='') as outfile:

            writer = csv.DictWriter(outfile, fieldnames=fieldnames)

            writer.writeheader()

            # Process each row in the input file

            for row in reader:

                text = row['text']

                words = text.split()  # Split the text into words

                for word_index, word in enumerate(words, start=1):  # Word count starts from 1

                    # Create a new row for each word

                    new_row = {

                        'id': row['id'],

                        'book': row['book'],

                        'chapter': row['chapter'],

                        'verse': row['verse'],

                        'word': word_index,

                        'text': word

                    }

                    # Write the new row to the output file

                    writer.writerow(new_row)

    Reading the Input File: The script opens the input CSV file using csv.DictReader to read the rows as dictionaries.

    Splitting the Text into Words: For each row, the text field is split into individual words.

    Creating New Rows: For each word in the text, a new row is created, including the original id, book, chapter, verse, the word index (count), and the word itself.

    Writing the Output File: The transformed data is written to a new CSV file using csv.DictWriter.

    id

    book

    chapter

    verse

    word (Word count within the verse)

    text (The word from the original text)

    # Regular expression to remove nikud (diacritics)

    nikud_pattern = r'[\u0591-\u05C7]'

    return re.sub(nikud_pattern, '', hebrew_text)

    with open(input_file, mode='r', encoding='utf-8') as infile:

        reader = csv.DictReader(infile)

        # Prepare the fieldnames for the output CSV

        fieldnames = ['id', 'book', 'chapter', 'verse', 'word', 'text']

        # Open the output file to write the transformed data

        with open(output_file, mode='w', encoding='utf-8', newline='') as outfile:

            writer = csv.DictWriter(outfile, fieldnames=fieldnames)

            writer.writeheader()

            # Process each row in the input file

            for row in reader:

                text = row['text']

                words = text.split()  # Split the text into words

                for word_index, word in enumerate(words, start=1):  # Word count starts from 1

                    # Remove nikud from Hebrew words

                    word_without_nikud = remove_nikud(word)

                    # Create a new row for each word

                    new_row = {

                        'id': row['id'],

                        'book': row['book'],

                        'chapter': row['chapter'],

                        'verse': row['verse'],

                        'word': word_index,

                        'text': word_without_nikud

                    }

                    # Write the new row to the output file

                    writer.writerow(new_row)

    Remove Nikud Function: A function remove_nikud() is added that uses a regular expression to remove Hebrew diacritical marks (nikud) from the words. The pattern [\u0591-\u05C7] captures the Unicode range for Hebrew nikud marks.

    Word Processing: Before adding each word to the new row in the CSV, the script passes the word through the remove_nikud() function to strip away any diacritics.

    Nikud Removal: The script strips nikud from each Hebrew word by using a regular expression to match and remove characters in the Unicode range for Hebrew diacritics (U+0591 to U+05C7).

    The rest of the logic remains the same, with each word being processed and outputted to the new CSV file, now without nikud.

    Identify Hebrew words within the text column.

    Create a new row for each Hebrew word, similar to how we were creating rows for each word before, but this time only for the Hebrew words.

    # Regular expression to remove nikud (diacritics)

    nikud_pattern = r'[\u0591-\u05C7]'

    return re.sub(nikud_pattern, '', hebrew_text)

    # Regular expression to find Hebrew words

    hebrew_pattern = r'[\u0590-\u05FF]+'

    return re.findall(hebrew_pattern, text)

    with open(input_file, mode='r', encoding='utf-8') as infile:

        reader = csv.DictReader(infile)

        # Prepare the fieldnames for the output CSV

        fieldnames = ['id', 'book', 'chapter', 'verse', 'word', 'text']

        # Open the output file to write the transformed data

        with open(output_file, mode='w', encoding='utf-8', newline='') as outfile:

            writer = csv.DictWriter(outfile, fieldnames=fieldnames)

            writer.writeheader()

            # Process each row in the input file

            for row in reader:

                text = row['text']

                hebrew_words = extract_hebrew_words(text)  # Extract Hebrew words

                for word_index, word in enumerate(hebrew_words, start=1):  # Word count starts from 1

                    # Remove nikud from Hebrew words

                    word_without_nikud = remove_nikud(word)

                    # Create a new row for each Hebrew word

                    new_row = {

                        'id': row['id'],

                        'book': row['book'],

                        'chapter': row['chapter'],

                        'verse': row['verse'],

                        'word': word_index,

                        'text': word_without_nikud

                    }

                    # Write the new row to the output file

                    writer.writerow(new_row)

    Extract Hebrew Words:

        The function extract_hebrew_words(text) uses a regular expression r'[\u0590-\u05FF]+' to match sequences of Hebrew characters in the text. This is done using re.findall(), which returns all the Hebrew words found in the text.

    Processing Hebrew Words:

        We now only process the Hebrew words found in the text. The script will create a new row for each Hebrew word, just as before, but this time only for words that match the Hebrew Unicode range.

    Nikud Removal:

        The script still removes nikud from each Hebrew word with the remove_nikud() function.

    Hebrew Word Extraction: The extract_hebrew_words() function finds all Hebrew words in the text column by looking for sequences of Hebrew characters (Unicode range \u0590-\u05FF).

    Word Processing: The script loops through the Hebrew words, removes nikud, and creates a row for each word.

    Writing Output: The output CSV file will contain a new row for each Hebrew word in the original text.

    The text will be parsed for Hebrew words and will create rows for each Hebrew word found.

    Words like "בְּרֵאשִׁית" and "הָאָרֶץ" will be handled, and if they have nikud, the nikud will be removed in the output.

    # Regular expression to remove nikud (diacritics)

    nikud_pattern = r'[\u0591-\u05C7]'

    return re.sub(nikud_pattern, '', hebrew_text)

    # Regular expression to match Hebrew words

    hebrew_pattern = r'[\u0590-\u05FF]+'

    return re.findall(hebrew_pattern, text)

    with open(input_file, mode='r', encoding='utf-8') as infile:

        reader = csv.DictReader(infile)

        # Prepare the fieldnames for the output CSV

        fieldnames = ['id', 'book', 'chapter', 'verse', 'word', 'text']

        # Open the output file to write the transformed data

        with open(output_file, mode='w', encoding='utf-8', newline='') as outfile:

            writer = csv.DictWriter(outfile, fieldnames=fieldnames)

            writer.writeheader()

            # Process each row in the input file

            for row in reader:

                text = row['text']

                hebrew_words = extract_hebrew_words_in_order(text)  # Extract Hebrew words in order

                for word_index, word in enumerate(hebrew_words, start=1):  # Word count starts from 1

                    # Remove nikud from Hebrew words

                    word_without_nikud = remove_nikud(word)

                    # Create a new row for each Hebrew word

                    new_row = {

                        'id': row['id'],

                        'book': row['book'],

                        'chapter': row['chapter'],

                        'verse': row['verse'],

                        'word': word_index,

                        'text': word_without_nikud

                    }

                    # Write the new row to the output file

                    writer.writerow(new_row)

    Preserving Word Order:

        The extract_hebrew_words_in_order() function now uses the regular expression r'[\u0590-\u05FF]+' to find and return Hebrew words in the order they appear in the text column. This ensures that we process Hebrew words in the exact sequence as they are found in the text, without any rearrangement.

    Hebrew Words Processing:

        After extracting the Hebrew words, we iterate over them in the order they were found, starting from word_index = 1 for the first Hebrew word in the verse. Each Hebrew word is stripped of nikud using the remove_nikud() function and written to the output file.

    Word Order in Output:

        The output file will have rows for each Hebrew word, and the words will appear in the same order as they appear in the original text, maintaining the correct sequence.

    Text Parsing: The script reads the text column, extracts Hebrew words (using the regex), and processes them in the order they appear.

    Hebrew Words: It then creates a new row for each Hebrew word, and the word column keeps track of the word's position in the verse.

    Nikud Removal: Nikud is removed from each Hebrew word before it's written to the output file.

    # Regular expression to remove nikud (diacritics)

    nikud_pattern = r'[\u0591-\u05C7]'

    return re.sub(nikud_pattern, '', hebrew_text)

    # Regular expression to match Hebrew words

    hebrew_pattern = r'[\u0590-\u05FF]+'

    return re.findall(hebrew_pattern, text)

    with open(input_file, mode='r', encoding='utf-8') as infile:

        reader = csv.DictReader(infile)

        # Prepare the fieldnames for the output CSV

        fieldnames = ['id', 'book', 'chapter', 'verse', 'word', 'text']

        # Open the output file to write the transformed data

        with open(output_file, mode='w', encoding='utf-8', newline='') as outfile:

            writer = csv.DictWriter(outfile, fieldnames=fieldnames)

            writer.writeheader()

            # Process each row in the input file

            for row in reader:

                text = row['text']

                hebrew_words = extract_hebrew_words_in_order(text)  # Extract Hebrew words in order

                for word_index, word in enumerate(hebrew_words, start=1):  # Word count starts from 1

                    # Remove nikud from Hebrew words

                    word_without_nikud = remove_nikud(word)

                    # Create a new row for each Hebrew word

                    new_row = {

                        'id': row['id'],

                        'book': row['book'],

                        'chapter': row['chapter'],

                        'verse': row['verse'],

                        'word': word_index,

                        'text': word_without_nikud

                    }

                    # Write the new row to the output file

                    writer.writerow(new_row)