Python file handling and data manipulation

File Handling Fundamentals

File handling is a crucial aspect of programming that allows you to store data persistently and read it back when needed.

File Operations Overview

• Reading: Accessing data from files

• Writing: Storing data to files

• Appending: Adding data to existing files

• Managing: Creating, deleting, moving files

File Modes in Python

Binary modes: Add 'b' (e.g., 'rb', 'wb')
Text modes: Default (e.g., 'r', 'w')

Basic File Operations

1. Reading Files

python

# Method 1: Using open() with close()
file = open('example.txt', 'r')
content = file.read()
file.close()

# Method 2: Using with statement (Recommended)
with open('example.txt', 'r') as file:
    content = file.read()
    print(content)

# Reading line by line
with open('example.txt', 'r') as file:
    for line in file:
        print(line.strip())  # strip() removes newline characters

# Reading all lines into a list
with open('example.txt', 'r') as file:
    lines = file.readlines()
    print(f"Total lines: {len(lines)}")
    for line_num, line in enumerate(lines, 1):
        print(f"Line {line_num}: {line.strip()}")

# Reading specific number of characters
with open('example.txt', 'r') as file:
    first_100_chars = file.read(100)
    print(f"First 100 characters: {first_100_chars}")

2. Writing Files

python

# Writing to a file (overwrites existing content)
with open('output.txt', 'w') as file:
    file.write("Hello, World!\n")
    file.write("This is a new line.\n")
    file.write("Python file handling is awesome!\n")

# Writing multiple lines at once
lines_to_write = [
    "Line 1: Python is powerful",
    "Line 2: File handling is essential", 
    "Line 3: Data persistence matters"
]

with open('multiple_lines.txt', 'w') as file:
    file.writelines(line + '\n' for line in lines_to_write)

# Using print function to write to file
with open('print_output.txt', 'w') as file:
    print("This goes to file", file=file)
    print("Multiple values:", 42, True, [1, 2, 3], file=file)

3. Appending to Files

python

x

# Writing to a file (overwrites existing content)
with open('output.txt', 'w') as file:
    file.write("Hello, World!\n")
    file.write("This is a new line.\n")
    file.write("Python file handling is awesome!\n")

# Writing multiple lines at once
lines_to_write = [
    "Line 1: Python is powerful",
    "Line 2: File handling is essential", 
    "Line 3: Data persistence matters"
]

with open('multiple_lines.txt', 'w') as file:
    file.writelines(line + '\n' for line in lines_to_write)

# Using print function to write to file
with open('print_output.txt', 'w') as file:
    print("This goes to file", file=file)
    print("Multiple values:", 42, True, [1, 2, 3], file=file)

File Handling Best Practices

Error Handling with Files

python

import os

def safe_file_operations(filename):
    """Demonstrate safe file operations with error handling"""
    
    # Check if file exists
    if not os.path.exists(filename):
        print(f"File '{filename}' does not exist. Creating it...")
    
    try:
        # Reading with error handling
        with open(filename, 'r') as file:
            content = file.read()
            print(f"File content:\n{content}")
            
    except FileNotFoundError:
        print(f"Error: File '{filename}' not found!")
    except PermissionError:
        print(f"Error: Permission denied for '{filename}'!")
    except IOError as e:
        print(f"IOError occurred: {e}")
    except Exception as e:
        print(f"Unexpected error: {e}")
    else:
        print("File read successfully!")
    finally:
        print("File operation completed.")

# Usage
safe_file_operations('example.txt')

# Check file information
filename = 'example.txt'
if os.path.exists(filename):
    print(f"File: {filename}")
    print(f"Size: {os.path.getsize(filename)} bytes")
    print(f"Last modified: {os.path.getmtime(filename)}")
else:
    print(f"File '{filename}' does not exist")

Working with Different File Formats

1. CSV Files (Comma-Separated Values)

python

import csv

# Writing to CSV file
def write_to_csv(filename, data):
    with open(filename, 'w', newline='') as csvfile:
        writer = csv.writer(csvfile)
        writer.writerow(['Name', 'Age', 'City', 'Salary'])  # Header
        writer.writerows(data)  # Multiple rows

# Sample data
employee_data = [
    ['John Doe', 30, 'New York', 50000],
    ['Jane Smith', 25, 'London', 45000],
    ['Bob Johnson', 35, 'Tokyo', 60000],
    ['Alice Brown', 28, 'Paris', 52000]
]

write_to_csv('employees.csv', employee_data)

# Reading from CSV file
def read_from_csv(filename):
    with open(filename, 'r') as csvfile:
        reader = csv.reader(csvfile)
        header = next(reader)  # Read header
        print(f"Header: {header}")
        
        employees = []
        for row_num, row in enumerate(reader, 1):
            employees.append({
                'name': row[0],
                'age': int(row[1]),
                'city': row[2],
                'salary': float(row[3])
            })
            print(f"Row {row_num}: {row}")
        
        return employees

employees = read_from_csv('employees.csv')

# Using DictReader for better column handling
with open('employees.csv', 'r') as csvfile:
    dict_reader = csv.DictReader(csvfile)
    for row in dict_reader:
        print(f"{row['Name']} from {row['City']} earns ${row['Salary']}")

2. JSON Files (JavaScript Object Notation)

python

import json

# Python data structure
company_data = {
    "company": "Tech Corp",
    "founded": 2010,
    "employees": [
        {
            "id": 1,
            "name": "John Doe",
            "department": "Engineering",
            "skills": ["Python", "JavaScript", "SQL"]
        },
        {
            "id": 2, 
            "name": "Jane Smith",
            "department": "Marketing",
            "skills": ["SEO", "Content Writing", "Analytics"]
        }
    ],
    "locations": ["New York", "London", "Tokyo"],
    "revenue": 1500000.50,
    "is_public": False
}

# Writing JSON to file
with open('company_data.json', 'w') as json_file:
    json.dump(company_data, json_file, indent=4)  # indent for pretty printing

print("JSON file created successfully!")

# Reading JSON from file
with open('company_data.json', 'r') as json_file:
    loaded_data = json.load(json_file)
    print("Company:", loaded_data['company'])
    print("Total employees:", len(loaded_data['employees']))
    
    for employee in loaded_data['employees']:
        print(f"- {employee['name']} ({employee['department']})")

# JSON string operations
json_string = json.dumps(company_data, indent=2)  # Convert to string
print("JSON String:")
print(json_string)

parsed_data = json.loads(json_string)  # Parse from string
print("Parsed data type:", type(parsed_data))

3. Text File Processing with Data Manipulation

python

def process_text_file(input_file, output_file):
    """Process text file: clean data, extract information, and create report"""
    
    with open(input_file, 'r') as infile:
        content = infile.read()
    
    # Data manipulation
    lines = content.split('\n')
    cleaned_lines = []
    
    word_count = 0
    char_count = 0
    number_count = 0
    
    for line in lines:
        if line.strip():  # Skip empty lines
            # Clean the line
            cleaned_line = line.strip()
            cleaned_lines.append(cleaned_line)
            
            # Count words and characters
            words = cleaned_line.split()
            word_count += len(words)
            char_count += len(cleaned_line)
            
            # Count numbers in the line
            for word in words:
                if word.replace('.', '').replace(',', '').isdigit():
                    number_count += 1
    
    # Create report
    report = f"""
    TEXT FILE ANALYSIS REPORT
    =========================
    Input File: {input_file}
    Output File: {output_file}
    
    Statistics:
    - Total Lines: {len(cleaned_lines)}
    - Total Words: {word_count} 
    - Total Characters: {char_count}
    - Numbers Found: {number_count}
    
    Processed Content:
    {''.join(f"{i+1:3d}: {line}\n" for i, line in enumerate(cleaned_lines))}
    """
    
    # Write processed data and report
    with open(output_file, 'w') as outfile:
        outfile.write(report)
    
    print(f"File processed successfully! Report saved to {output_file}")

# Create sample input file
sample_content = """Hello World! This is a sample text file.
It contains 123 numbers and various words.
Python programming is fun and 456 exciting.
Let's process this 789 data efficiently.

Empty lines should be ignored.

Final line with 1000 as a number."""

with open('sample_input.txt', 'w') as f:
    f.write(sample_content)

# Process the file
process_text_file('sample_input.txt', 'analysis_report.txt')

Advanced File Operations

Binary File Handling

python

def binary_file_operations():
    """Demonstrate binary file operations"""
    
    # Writing binary data
    binary_data = bytes(range(256))  # 0-255 bytes
    
    with open('binary_data.bin', 'wb') as bin_file:
        bin_file.write(binary_data)
    
    print("Binary file created with 256 bytes")
    
    # Reading binary data
    with open('binary_data.bin', 'rb') as bin_file:
        read_data = bin_file.read()
        print(f"Read {len(read_data)} bytes from binary file")
        
        # Display first 20 bytes as hex
        print("First 20 bytes (hex):", ' '.join(f'{b:02x}' for b in read_data[:20]))
    
    # Working with images (copy operation)
    try:
        with open('binary_data.bin', 'rb') as source:
            with open('binary_copy.bin', 'wb') as destination:
                # Read and write in chunks for large files
                chunk_size = 1024
                while True:
                    chunk = source.read(chunk_size)
                    if not chunk:
                        break
                    destination.write(chunk)
        print("Binary file copied successfully!")
    except Exception as e:
        print(f"Error copying file: {e}")

binary_file_operations()

File Searching and Pattern Matching

python

import re
import os

def search_in_files(directory, pattern, file_extension='.txt'):
    """Search for pattern in all files with given extension in directory"""
    
    results = {}
    
    for filename in os.listdir(directory):
        if filename.endswith(file_extension):
            filepath = os.path.join(directory, filename)
            
            try:
                with open(filepath, 'r', encoding='utf-8') as file:
                    content = file.read()
                    
                    # Search for pattern using regex
                    matches = re.findall(pattern, content, re.IGNORECASE)
                    
                    if matches:
                        results[filename] = {
                            'match_count': len(matches),
                            'matches': matches,
                            'file_size': os.path.getsize(filepath)
                        }
                        
            except Exception as e:
                print(f"Error processing {filename}: {e}")
    
    return results

def create_sample_files():
    """Create sample files for testing search functionality"""
    os.makedirs('search_test', exist_ok=True)
    
    files_content = {
        'file1.txt': 'Python is a programming language. Python is versatile.',
        'file2.txt': 'Java and Python are both popular languages.',
        'file3.txt': 'This file contains the word python multiple times. Python!',
        'file4.txt': 'No relevant content here. Just random text.'
    }
    
    for filename, content in files_content.items():
        with open(f'search_test/{filename}', 'w') as f:
            f.write(content)

# Create test files
create_sample_files()

# Search for 'python' in text files
search_results = search_in_files('search_test', r'python', '.txt')

print("SEARCH RESULTS:")
print("=" * 50)
for filename, result in search_results.items():
    print(f"File: {filename}")
    print(f"  Matches: {result['match_count']}")
    print(f"  Found: {', '.join(result['matches'])}")
    print(f"  File size: {result['file_size']} bytes")
    print("-" * 30)

Data Manipulation Techniques

1. Data Cleaning and Transformation

python

def clean_and_transform_data(input_file, output_file):
    """Clean and transform data from input file to output file"""
    
    with open(input_file, 'r') as infile:
        raw_data = infile.readlines()
    
    cleaned_data = []
    
    for line_num, line in enumerate(raw_data, 1):
        # Remove whitespace and convert to lowercase
        cleaned_line = line.strip().lower()
        
        if cleaned_line:  # Only process non-empty lines
            # Split into words and clean each word
            words = cleaned_line.split()
            cleaned_words = []
            
            for word in words:
                # Remove punctuation and special characters
                cleaned_word = ''.join(char for char in word if char.isalnum())
                if cleaned_word:  # Only add non-empty words
                    cleaned_words.append(cleaned_word)
            
            # Join back into a cleaned line
            if cleaned_words:
                cleaned_line = ' '.join(cleaned_words)
                cleaned_data.append(cleaned_line)
    
    # Write cleaned data to output file
    with open(output_file, 'w') as outfile:
        outfile.write('\n'.join(cleaned_data))
    
    # Generate statistics
    original_lines = len(raw_data)
    cleaned_lines = len(cleaned_data)
    original_words = sum(len(line.split()) for line in raw_data)
    cleaned_words = sum(len(line.split()) for line in cleaned_data)
    
    stats = f"""
    DATA CLEANING REPORT
    ====================
    Original file: {input_file}
    Cleaned file: {output_file}
    
    Statistics:
    - Original lines: {original_lines}
    - Cleaned lines: {cleaned_lines}
    - Lines removed: {original_lines - cleaned_lines}
    - Original words: {original_words}
    - Cleaned words: {cleaned_words}
    - Words removed: {original_words - cleaned_words}
    
    Cleaning operations performed:
    1. Removed empty lines
    2. Converted to lowercase
    3. Removed punctuation and special characters
    4. Removed extra whitespace
    """
    
    print(stats)
    return cleaned_data

# Create sample dirty data
dirty_data = """Hello, World! This is a TEST.
Multiple   spaces    here...
SPECIAL @#$% CHARACTERS!!!
Empty line above and below:

Python & Java Programming.
Numbers: 123, 456, 789!
Final Line."""

with open('dirty_data.txt', 'w') as f:
    f.write(dirty_data)

# Clean the data
cleaned = clean_and_transform_data('dirty_data.txt', 'cleaned_data.txt')

2. Data Analysis with File Operations

python

def analyze_text_data(filename):
    """Perform comprehensive analysis on text data"""
    
    with open(filename, 'r') as file:
        content = file.read()
    
    # Basic statistics
    lines = content.split('\n')
    words = content.split()
    characters = list(content)
    
    # Word frequency analysis
    word_freq = {}
    for word in words:
        clean_word = word.lower().strip('.,!?;:"()')
        if clean_word:
            word_freq[clean_word] = word_freq.get(clean_word, 0) + 1
    
    # Character frequency
    char_freq = {}
    for char in content:
        if char.isalpha():  # Only count letters
            lower_char = char.lower()
            char_freq[lower_char] = char_freq.get(lower_char, 0) + 1
    
    # Sentence count (rough estimate)
    sentences = content.replace('!', '.').replace('?', '.').split('.')
    sentence_count = len([s for s in sentences if s.strip()])
    
    # Generate report
    analysis_report = f"""
    TEXT DATA ANALYSIS
    ==================
    File: {filename}
    
    BASIC STATISTICS:
    - Total Lines: {len(lines)}
    - Total Words: {len(words)}
    - Total Characters: {len(characters)}
    - Total Sentences: {sentence_count}
    - Average Words per Sentence: {len(words)/sentence_count:.2f}
    - Average Word Length: {sum(len(word) for word in words)/len(words):.2f}
    
    MOST FREQUENT WORDS (Top 10):
    {''.join(f'    {word}: {count}\n' for word, count in sorted(word_freq.items(), key=lambda x: x[1], reverse=True)[:10])}
    
    CHARACTER DISTRIBUTION (Top 10):
    {''.join(f'    {char}: {count}\n' for char, count in sorted(char_freq.items(), key=lambda x: x[1], reverse=True)[:10])}
    """
    
    # Write analysis to file
    report_filename = f"analysis_{filename}"
    with open(report_filename, 'w') as report_file:
        report_file.write(analysis_report)
    
    print(f"Analysis complete! Report saved to {report_filename}")
    return analysis_report

# Create a sample file for analysis
sample_text = """Python is a powerful programming language. It is easy to learn and use.
Python has many libraries for data analysis, web development, and machine learning.
The language is versatile and can be used for various applications.
Many companies use Python for their projects and products.
Learning Python can open many doors in the tech industry."""

with open('sample_analysis.txt', 'w') as f:
    f.write(sample_text)

# Analyze the text
analysis = analyze_text_data('sample_analysis.txt')
print(analysis)

Practical Example: Student Management System

python

import json
import csv
import os
from datetime import datetime

class StudentManager:
    def __init__(self, data_file='students.json'):
        self.data_file = data_file
        self.students = self.load_data()
    
    def load_data(self):
        """Load student data from JSON file"""
        if os.path.exists(self.data_file):
            try:
                with open(self.data_file, 'r') as file:
                    return json.load(file)
            except (json.JSONDecodeError, IOError):
                return {}
        return {}
    
    def save_data(self):
        """Save student data to JSON file"""
        try:
            with open(self.data_file, 'w') as file:
                json.dump(self.students, file, indent=4)
            return True
        except IOError as e:
            print(f"Error saving data: {e}")
            return False
    
    def add_student(self, student_id, name, grade, subjects):
        """Add a new student"""
        if student_id in self.students:
            print(f"Student ID {student_id} already exists!")
            return False
        
        self.students[student_id] = {
            'name': name,
            'grade': grade,
            'subjects': subjects,
            'created_at': datetime.now().isoformat(),
            'grades': {}
        }
        return self.save_data()
    
    def update_grade(self, student_id, subject, grade):
        """Update student grade"""
        if student_id not in self.students:
            print(f"Student ID {student_id} not found!")
            return False
        
        self.students[student_id]['grades'][subject] = grade
        return self.save_data()
    
    def export_to_csv(self, csv_filename='students_export.csv'):
        """Export student data to CSV"""
        try:
            with open(csv_filename, 'w', newline='') as csvfile:
                fieldnames = ['Student_ID', 'Name', 'Grade', 'Subjects', 'Average_Grade']
                writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
                
                writer.writeheader()
                for student_id, data in self.students.items():
                    # Calculate average grade
                    grades = data['grades'].values()
                    avg_grade = sum(grades) / len(grades) if grades else 0
                    
                    writer.writerow({
                        'Student_ID': student_id,
                        'Name': data['name'],
                        'Grade': data['grade'],
                        'Subjects': ', '.join(data['subjects']),
                        'Average_Grade': f"{avg_grade:.2f}"
                    })
            print(f"Data exported to {csv_filename}")
            return True
        except Exception as e:
            print(f"Error exporting to CSV: {e}")
            return False
    
    def generate_report(self, report_filename='student_report.txt'):
        """Generate a comprehensive student report"""
        report_lines = []
        report_lines.append("STUDENT MANAGEMENT SYSTEM REPORT")
        report_lines.append("=" * 50)
        report_lines.append(f"Generated on: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
        report_lines.append(f"Total Students: {len(self.students)}")
        report_lines.append("")
        
        for student_id, data in self.students.items():
            report_lines.append(f"Student ID: {student_id}")
            report_lines.append(f"Name: {data['name']}")
            report_lines.append(f"Grade: {data['grade']}")
            report_lines.append(f"Subjects: {', '.join(data['subjects'])}")
            
            if data['grades']:
                report_lines.append("Grades:")
                for subject, grade in data['grades'].items():
                    report_lines.append(f"  - {subject}: {grade}")
                avg_grade = sum(data['grades'].values()) / len(data['grades'])
                report_lines.append(f"Average Grade: {avg_grade:.2f}")
            else:
                report_lines.append("No grades recorded yet.")
            
            report_lines.append("-" * 30)
        
        # Write report to file
        with open(report_filename, 'w') as report_file:
            report_file.write('\n'.join(report_lines))
        
        print(f"Report generated: {report_filename}")
        return report_lines

# Demonstration
def demonstrate_student_manager():
    """Demonstrate the StudentManager class"""
    
    manager = StudentManager()
    
    # Add some students
    manager.add_student('S001', 'Alice Johnson', '10th', ['Math', 'Science', 'English'])
    manager.add_student('S002', 'Bob Smith', '11th', ['History', 'Art', 'Math'])
    manager.add_student('S003', 'Carol Davis', '10th', ['Science', 'Math', 'PE'])
    
    # Update grades
    manager.update_grade('S001', 'Math', 95)
    manager.update_grade('S001', 'Science', 88)
    manager.update_grade('S001', 'English', 92)
    
    manager.update_grade('S002', 'History', 78)
    manager.update_grade('S002', 'Art', 85)
    manager.update_grade('S002', 'Math', 90)
    
    # Export and generate reports
    manager.export_to_csv()
    manager.generate_report()
    
    print("Student management system demonstration completed!")
    print("Files created:")
    print("- students.json (main data file)")
    print("- students_export.csv (CSV export)")
    print("- student_report.txt (detailed report)")

# Run demonstration
demonstrate_student_manager()

Key Takeaways

1. Always use with statement for automatic file closing

2. Handle exceptions for robust file operations

3. Choose appropriate file modes based on your needs

4. Use proper encoding for text files

5. Process large files in chunks to avoid memory issues

6. Validate data when reading from files

7. Backup important files before processing

8. Use appropriate file formats (JSON for structured data, CSV for tabular data, etc.)

This extensive guide covers all essential aspects of file handling and data manipulation in Python, from basic operations to advanced techniques used in real-world applications.