data_processing.ipynb

{
    "cells": [
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "# Data downloading\n",
                "\n",
                "## 1.Downing allosteric site data from shsmu.edu.cn\n",
                "\n",
                "> `data/allosteric_site_shsmu.json` \n",
                "> \n",
                "> contains data crawling form shsmu.edu.cn(home->featrue->site)."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 6,
            "metadata": {},
            "outputs": [
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "100%|██████████| 230/230 [00:01<00:00, 115.41it/s]\n"
                    ]
                }
            ],
            "source": [
                "# there are 1928 allosteric site in shsmu, but 1928th 3GVU can not download.\n",
                "\n",
                "from utils.data_download import download_shsmu_as, download_rcsb, unzip\n",
                "\n",
                "allosteric_site_index = 'data/allosteric_site_shsmu.json'\n",
                "\n",
                "download_shsmu_as(allosteric_site_path=allosteric_site_index, outpath='data/shsmu_allosteric_site/')\n",
                "unzip(origin_dir='data/shsmu_allosteric_site/', outdir='data/shsmu_allosteric_site/allosteric_site/')\n",
                "\n",
                "!rm -r data/shsmu_allosteric_site/*.pdb.gz\n",
                "\n",
                "download_rcsb(allosteric_site_path=allosteric_site_index, outpath='data/shsmu_allosteric_site/')\n",
                "\n",
                "!rm -r data/shsmu_allosteric_site/3gvu.pdb.gz\n",
                "\n",
                "unzip(origin_dir='data/shsmu_allosteric_site/', outdir='data/shsmu_allosteric_site/rscb_pdb/')\n",
                "\n",
                "!rm -r data/shsmu_allosteric_site/*.pdb.gz"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "> `data/ASD_Release_201909_AS.txt` \n",
                ">\n",
                "> is the description file of the allostericsite, but hard to read."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "from utils.data_process import transform_txt_to_csv\n",
                "\n",
                "txt_path = 'data/ASD_Release_201909_AS.txt'\n",
                "csv_path = 'data/ASD_Release_201909_AS.csv'\n",
                "transform_txt_to_csv(path=txt_path, outpath=csv_path)"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## 2.Download pretain data form rcsb\n",
                "\n",
                "> `data/pretrain/list_file_protein_xray_max3A_total.txt`\n",
                ">\n",
                "> includes pdbids with resolution of 0.5 ~ 3A obtained by X-Ray experiment."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "# download logs are stored in succeed.log, timeout.log, wrang.log\n",
                "\n",
                "from utils.data_download import download_pretarining_data, redownload_error_pretarining_data\n",
                "\n",
                "pdblist = 'test.txt'\n",
                "output_path = 'data/allo2023_pdb_suplement/'\n",
                "\n",
                "download_pretarining_data(list_path=pdblist, outpath=output_path)"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "> If there are links in timeout.log or wrang.log run redownload_error_pretarining_data() to redownload failed pdbs."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "timeout_path = 'data/pretrain/timeout.log'\n",
                "wrang_path = 'data/pretrain/wrang.log'\n",
                "\n",
                "redownload_error_pretarining_data(timeout_path, wrang_path, output_path)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "from utils.data_download import unzip\n",
                "\n",
                "dir_path = '/mnt/g/Little-LL/pretrain/'\n",
                "out_path = '/mnt/g/Little-LL/pretrain_pdb/'\n",
                "\n",
                "unzip(dir_path, out_path)"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "# Data processing"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## 1.Build a BERT tokenizer of residue"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "# optional step\n",
                "\n",
                "from utils.data_process import build_tokenizer_dataset, tokenizer_json_to_txt\n",
                "\n",
                "pdb_paths = '/mnt/g/Little-LL/pretrain_pdb/'\n",
                "output_path ='/mnt/g/Little-LL/pretrain_tokenizer/'\n",
                "\n",
                "build_tokenizer_dataset(path=pdb_paths, outpath=output_path)\n",
                "tokenizer_json_to_txt(output_path)"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "cd `models/` and run `python tokenizer_building.py`\n",
                "\n",
                "the output tokenizer config is in `models/tokenizer/` "
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## 2.Build the allosteric site dataset"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "from utils.data_process import build_allosteric_dataset\n",
                "\n",
                "# origin pdb file to input data (json)\n",
                "pdb_dir = 'data/shsmu_allosteric_site/rcsb_pdb/'\n",
                "out_dir = 'data/allosteric_site/input/'\n",
                "build_allosteric_dataset(pdb_dir, out_dir, save_repeat_chhain=True)\n",
                "\n",
                "pdb_dir = 'data/shsmu_allosteric_site/allosteric_site/'\n",
                "out_dir = 'data/allosteric_site/target/'\n",
                "build_allosteric_dataset(pdb_dir, out_dir, save_repeat_chhain=True)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "from utils.pre_data import pre_single_a, pre_data\n",
                "\n",
                "as_path = 'data/allosteric_site/target/'\n",
                "pdb_path = 'data/allosteric_site/input/'\n",
                "data_json = 'data/allosteric_site/data_all.json'\n",
                "\n",
                "# pre_single_a(target_dir=as_path, pdb_dir=pdb_path, output_json=data_json)\n",
                "pre_data(target_dir=as_path, pdb_dir=pdb_path, output_json=data_json)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "from utils.pre_data import transform_data, split_train_test\n",
                "\n",
                "data_json = 'data/allosteric_site/data_all.json'\n",
                "data_test = 'data/allosteric_site/data_test.json'\n",
                "data_train = 'data/allosteric_site/data_train.json'\n",
                "\n",
                "inputs, targets = transform_data(data_path=data_json)\n",
                "\n",
                "split_train_test(inputs, targets, train_file=data_train, test_file=data_test)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## 3.Build the dataset for pretrain ResidueRobertaMLM"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "> Transform .pdb files to json. (Do not save the same chains in a pdb)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "from utils.data_process import build_allosteric_dataset\n",
                "\n",
                "# origin pdb file to input data (json)\n",
                "pdb_dir = '/mnt/g/Little-LL/pretrain_pdb/'\n",
                "out_dir = '/mnt/g/Little-LL/pretrain_input/'\n",
                "build_allosteric_dataset(pdb_dir, out_dir, save_repeat_chhain=False)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "from utils.pre_data import pre_data_rcsb\n",
                "\n",
                "rcsb_path = '/mnt/g/Little-LL/pretrain_input/'\n",
                "rcsb_json = '/mnt/g/Little-LL/rcsb_all.json'\n",
                "\n",
                "pre_data_rcsb(rcsb_dir=rcsb_path, output_json=rcsb_json, split=10000)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "from utils.pre_data import transform_pretrain_data\n",
                "\n",
                "rcsb_json_dir = '/mnt/g/Little-LL/pretrain_rcsb_all/'\n",
                "rcsb_input_dir = '/mnt/g/Little-LL/pretrain_rcsb_inputs/'\n",
                "\n",
                "transform_pretrain_data(rcsb_json_dir, rcsb_input_dir)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## 4.Pocket detecting"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "from utils.data_process import extract_from_fpocket\n",
                "\n",
                "fpocket_path = '/home/little-ll/Demo/python/allosteric-site/data/shsmu_allosteric_site/rscb_pdb/fpocket_out_pocketPDB'\n",
                "npy_save_path = '/home/little-ll/Demo/python/allosteric-site/data/shsmu_allosteric_site/rscb_pdb/fpocket_out_pocketNPY'\n",
                "\n",
                "pockets_all = extract_from_fpocket(fpocket_path=fpocket_path, save_path=npy_save_path)\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "import numpy as np\n",
                "import operator\n",
                "\n",
                "path = '/home/little-ll/Demo/python/allosteric-site/data/shsmu_allosteric_site/rscb_pdb/fpocket_out_pocketNPY/1A3W_out.npy'\n",
                "load_dict = np.load(path, allow_pickle=True).item()\n",
                "print(load_dict['Pockets'])"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "# Test"
            ]
        }
    ],
    "metadata": {
        "kernelspec": {
            "display_name": "pytorch",
            "language": "python",
            "name": "python3"
        },
        "language_info": {
            "codemirror_mode": {
                "name": "ipython",
                "version": 3
            },
            "file_extension": ".py",
            "mimetype": "text/x-python",
            "name": "python",
            "nbconvert_exporter": "python",
            "pygments_lexer": "ipython3",
            "version": "3.10.4"
        },
        "orig_nbformat": 4
    },
    "nbformat": 4,
    "nbformat_minor": 2
}