PYTHON

droping Duplicates

# this is based on 2 factors the name of the dog and the breed of the dog so we can
# have 2 dog with the same name but diff breed.

df.drop_duplicates(subset=["name", "breed"])

# Without including index
df.drop_duplicates(subset=["name", "breed"], index = False)

Duplicate Remove

from pathlib import Path
import hashlib
import os

def remove_duplicate(path):
    unique = {}
    for file in Path(path).rglob('*'):
        if file.is_file():
            with open(file, 'rb') as f:
                filehash = hashlib.md5(f.read()).hexdigest()
                if filehash not in unique:
                    unique[filehash] = file
                else:
                    # Test print before removing
                    print(f'Removing --> {unique[filehash]}')
                    #os.remove(unique[filehash])

if __name__ == '__main__':
    path = r'C:foo'
    remove_duplicate(path)