How to keep the result reproducable?

[Whisht]

I built a language classification task with custom dataset and a textcnn model. When I run fit() and test() twice, the loss and accuracy are different between these two experiments. I fixed the random seed as 2021 in dataset producing procedure by train_test_split(random_state=2021). And I fixed the same random seed for other config with pl.seed_everything(2021). But the result still changes.

def get_train_test_dataloader(
    df, val_size=0.2, test_size=0.2, batch_size=128, num_workers=20
):
    # all_data = pd.read_feather(df_path)
    # if name:
    #     all_data = all_data[all_data["company"] == name]
    train_df, test_df = train_test_split(
        df, test_size=test_size + val_size, random_state=2021
    )
    val_df, test_df = train_test_split(
        test_df, test_size=val_size / (val_size + test_size), random_state=2021
    )
    train_dataset = SafetyDataset(train_df)
    test_dataset = SafetyDataset(test_df)
    val_dataset = SafetyDataset(val_df)
    train_dataloader = DataLoader(
        train_dataset,
        batch_size,
        shuffle=True,
        num_workers=num_workers,
        pin_memory=True,
    )
    test_dataloader = DataLoader(
        test_dataset, batch_size, num_workers=num_workers, pin_memory=True
    )
    val_dataloader = DataLoader(
        val_dataset, batch_size, num_workers=num_workers, pin_memory=True
    )
    return train_dataloader, test_dataloader, val_dataloader

class PLTextCNN(pl.LightningModule):
    def __init__(
        self,
        n_tokens=3502,
        n_embedding=32,
        n_filters=256,
        filter_sizes=(2, 3, 4),
        name="TextCNN",
        dropout=0.4,
        n_classes=2,
    ):
        self.name = name
        super().__init__()
        self.embedding = nn.Embedding(n_tokens, n_embedding, padding_idx=0)
        self.convs = nn.ModuleList(
            [nn.Conv2d(1, n_filters, (k, n_embedding)) for k in filter_sizes]
        )
        # self.norm = nn.BatchNorm1d(n_filters * len(filter_sizes), n_classes)
        # self.norm = nn.LayerNorm()
        self.dropout = nn.Dropout(dropout)
        self.fc = nn.Linear(n_filters * len(filter_sizes), n_classes)
        self.criteria = nn.CrossEntropyLoss()
        self.metrics = nn.ModuleDict(
            {
                "accuracy": pl.metrics.Accuracy(),
                "confusion_matrix": pl.metrics.ConfusionMatrix(num_classes=n_classes),
                # "f1": pl.metrics.classification.F1(num_classes=n_classes),
                # "f1": f1_score,
            }
        )

    @staticmethod
    def conv_and_pool(x, conv):
        x = f.relu(conv(x)).squeeze(3)
        x = f.max_pool1d(x, x.size(2)).squeeze(2)
        return x

    def forward(self, x):
        out = self.embedding(x)
        out = out.unsqueeze(dim=1)
        # out = out.reshape((1, 1, *out.shape))
        out = torch.cat([self.conv_and_pool(out, conv) for conv in self.convs], 1)
        # out = self.norm(out)
        out = self.dropout(out)
        out = self.fc(out)
        return out

    def training_step(self, batch, batch_idx):
        x, y = batch
        x = x.view(x.size(0), -1)
        y_hat = self(x)
        loss = self.criteria(y_hat, y)
        self.log("train_loss", loss)
        self.log(
            "train_f1",
            f1_score(
                y.cpu().numpy(),
                torch.argmax(f.softmax(y_hat, dim=1), dim=1).cpu().numpy(),
            ),
            prog_bar=True,
        )
        return loss

    def configure_optimizers(self):
        optimizer = torch.optim.SGD(self.parameters(), lr=1e-2)
        scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
            optimizer,
        )

        return {
            "optimizer": optimizer,
            "lr_scheduler": scheduler,
            "monitor": "val_f1",
        }

    def test_step(self, batch, batch_idx):
        x, y = batch
        x = x.view(x.size(0), -1)
        y_hat = self(x)
        loss = self.criteria(y_hat, y)
        self.log("test_loss", loss)
        self.log("test_accuracy", self.metrics["accuracy"](y_hat, y))
        self.log(
            "test_f1",
            f1_score(
                y.cpu().numpy(),
                torch.argmax(f.softmax(y_hat, dim=1), dim=1).cpu().numpy(),
            ),
        )

    def validation_step(self, batch, batch_idx):
        x, y = batch
        x = x.view(x.size(0), -1)
        y_hat = self(x)
        loss = self.criteria(y_hat, y)
        self.log("val_loss", loss)
        self.log(
            "val_f1",
            f1_score(
                y.cpu().numpy(),
                torch.argmax(f.softmax(y_hat, dim=1), dim=1).cpu().numpy(),
            ),
        )
        self.log("val_accuracy", self.metrics["accuracy"](y_hat, y))
        return loss

def run():

    pl.seed_everything(2021)

    df_all = pd.read_feather("data/data.feather")
    train, test, val = get_train_test_dataloader(df_all)

    model = PLTextCNN(n_tokens=len(train.dataset.vector))
    trainer = pl.Trainer(
        gpus=1,
        max_epochs=10,
        progress_bar_refresh_rate=10,
    )
    trainer.fit(model, train_dataloader=train, val_dataloaders=val)
    trainer.test(model, test)

[ankur56]

If you are training on GPU(s), you also need to include "deterministic=True" in your trainer. Although, in my experience, even that may not be sufficient to ensure reproducibility for some models.

[Whisht]

Thanks. Could you explain how this parameter work on GPU setting? By controlling the batch order?

[ankur56]

From what I know, batch order is already deterministic since "shuffle=False" by default in the DataLoader. "deterministic=True" tells CuDNN to only use deterministic algorithms for all the operations at the cost of some performance degradation. See the following links,
https://pytorch-lightning.readthedocs.io/en/0.7.6/trainer.html#reproducibility
https://stackoverflow.com/questions/56354461/reproducibility-and-performance-in-pytorch