c44090cc-1b99-4c95-8624-38fb4b5834f9.txt

import os
import sys
with open(sys.argv[0]) as f:
    code = f.read() # read the code of this file ASAP, for logging
import uuid
import time
from dataclasses import dataclass
from functools import lru_cache
from pathlib import Path

os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
import torch
torch.empty(1, device="cuda", requires_grad=True).backward() # prevents a bug on some systems
from torch import Tensor, nn
import torch.nn.functional as F
import torch.distributed as dist
# use of FlexAttention contributed by @KoszarskyB
from torch.nn.attention.flex_attention import BlockMask, flex_attention
torch._inductor.config.coordinate_descent_tuning = True # turn this off for a faster compile time (but slightly slower run)

# -----------------------------------------------------------------------------
# Custom operators : FP8 matmul by @YouJiacheng

@torch.library.custom_op("nanogpt::mm", mutates_args=())
def mm_op(x: Tensor, w: Tensor, x_s: float, w_s: float, grad_s: float) -> tuple[Tensor, Tensor, Tensor]:
    @torch.compile
    def impl(x: Tensor, w: Tensor):
        assert x.is_contiguous() and w.is_contiguous()
        x_f8 = x.mul(x_s).to(torch.float8_e4m3fn)
        w_f8 = w.mul(w_s).to(torch.float8_e4m3fn)
        out = torch._scaled_mm(
            x_f8,
            w_f8.t(),
            out_dtype=torch.bfloat16,
            scale_a=x.new_tensor(1 / x_s, dtype=torch.float32),
            scale_b=x.new_tensor(1 / w_s, dtype=torch.float32),
            use_fast_accum=True,
        )
        return out, x_f8, w_f8

    return impl(x, w)

@mm_op.register_fake
def _(x: Tensor, w: Tensor, *_):
    assert x.ndim == w.ndim == 2
    assert x.shape[1] == w.shape[1]
    assert x.device == w.device
    assert x.is_contiguous() and w.is_contiguous()
    return x @ w.t(), x.to(torch.float8_e4m3fn), w.to(torch.float8_e4m3fn)

@torch.library.custom_op("nanogpt::mm_backward", mutates_args=())
def mm_backward_op(g: Tensor, x_f8: Tensor, w_f8: Tensor, x_s: float, w_s: float, grad_s: float) -> tuple[Tensor, Tensor]:
    @torch.compile
    def impl(grad: Tensor, x_f8: Tensor, w_f8: Tensor):
        assert grad.is_contiguous()
        x_inv_s = grad.new_tensor(1 / x_s, dtype=torch.float32)
        w_inv_s = grad.new_tensor(1 / w_s, dtype=torch.float32)
        grad_inv_s = grad.new_tensor(1 / grad_s, dtype=torch.float32)
        grad_f8 = grad.mul(grad_s).to(torch.float8_e5m2)
        grad_x = torch._scaled_mm(
            grad_f8,
            w_f8.t().contiguous().t(),
            out_dtype=torch.bfloat16,
            scale_a=grad_inv_s,
            scale_b=w_inv_s,
            use_fast_accum=False,
        )
        # faster than grad_f8_t @ x_f8, for (d_out, d_in) == (50304, 768)
        grad_w = torch._scaled_mm(
            x_f8.t().contiguous(),
            grad_f8.t().contiguous().t(),
            out_dtype=torch.float32,
            scale_a=x_inv_s,
            scale_b=grad_inv_s,
            use_fast_accum=False,
        ).t()
        return grad_x, grad_w

    return impl(g, x_f8, w_f8)

@mm_backward_op.register_fake
def _(g: Tensor, x_f8: Tensor, w_f8: Tensor, *_):
    return x_f8.to(torch.bfloat16), w_f8.to(torch.float32)

def backward(ctx, grad_out: Tensor, *_):
    x_f8, w_f8 = ctx.saved_tensors
    x_s, w_s, grad_s = ctx.scales
    grad_x, grad_w = torch.ops.nanogpt.mm_backward(
        grad_out, x_f8, w_f8, x_s, w_s, grad_s
    )
    return grad_x, grad_w, None, None, None

def setup_context(ctx: torch.autograd.function.FunctionCtx, inputs, output):
    *_, x_s, w_s, grad_s = inputs
    _, x_f8, w_f8 = output
    ctx.save_for_backward(x_f8, w_f8)
    ctx.scales = x_s, w_s, grad_s
    ctx.set_materialize_grads(False)

mm_op.register_autograd(backward, setup_context=setup_context)

# -----------------------------------------------------------------------------
# Muon optimizer

@torch.compile
def zeropower_via_newtonschulz5(G: Tensor, steps: int) -> Tensor:
    """
    Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
    quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
    of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
    zero even beyond the point where the iteration no longer converges all the way to one everywhere
    on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
    where S' is diagonal with S_{ii}' ~ Uniform(0.5, 1.5), which turns out not to hurt model
    performance at all relative to UV^T, where USV^T = G is the SVD.
    """
    assert G.ndim >= 2 # batched Muon implementation by @scottjmaddox, and put into practice in the record by @YouJiacheng
    a, b, c = (3.4445, -4.7750,  2.0315)
    X = G.bfloat16()
    if G.size(-2) > G.size(-1):
        X = X.mT

    # Ensure spectral norm is at most 1
    X = X / (X.norm(dim=(-2, -1), keepdim=True) + 1e-7)
    # Perform the NS iterations
    for _ in range(steps):
        A = X @ X.mT
        B = b * A + c * A @ A # quintic computation strategy adapted from suggestion by @jxbz, @leloykun, and @YouJiacheng
        X = a * X + B @ X
    
    if G.size(-2) > G.size(-1):
        X = X.mT
    return X

class Muon(torch.optim.Optimizer):
    """
    Muon - MomentUm Orthogonalized by Newton-schulz

    Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-
    processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
    matrix. To efficiently orthogonalize each update, we use a Newton-Schulz iteration, which has
    the advantage that it can be stably run in bfloat16 on the GPU.

    Some warnings:
    - This optimizer assumes that all parameters passed in are 2D.
    - It should not be used for the embedding layer, the final fully connected layer, or any {0,1}-D
    parameters; those should all be optimized by a standard method (e.g., AdamW).
    - To use it with 4D convolutional filters, it works well to just flatten their last 3 dimensions.
    - We believe it is unlikely to work well for training with small batch size.
    - We believe it may not work well for finetuning pretrained models, but we haven"t tested this.
    - We have not yet tried this optimizer for training scenarios larger than NanoGPT (124M).

    Arguments:
        lr: The learning rate used by the internal SGD.
        momentum: The momentum used by the internal SGD.
        nesterov: Whether to use Nesterov-style momentum in the internal SGD. (recommended)
        ns_steps: The number of Newton-Schulz iteration steps to use.
    """
    def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True, ns_steps=5, rank=0, world_size=1):
        self.rank = rank
        self.world_size = world_size
        defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, ns_steps=ns_steps)
        params: list[Tensor] = [*params]
        assert all(isinstance(p, Tensor) for p in params)
        sizes = {p.numel() for p in params}
        def create_update_buffer(size: int):
            b = torch.empty(self.world_size, size, dtype=torch.bfloat16, device="cuda")
            return dict(update_buffer=b, update_buffer_views=[b[i] for i in range(self.world_size)])
        param_groups = [
            dict(params=[p for p in params if p.numel() == size], **create_update_buffer(size)) for size in sizes]
        super().__init__(param_groups, defaults)

    @torch.no_grad()
    def step(self):
        for group in self.param_groups:
            lr = group["lr"]
            momentum = group["momentum"]
            nesterov = group["nesterov"]
            ns_steps = group["ns_steps"]
            update_buffer = group["update_buffer"]
            update_buffer_views: list[Tensor] = group["update_buffer_views"]
            # generate weight updates in distributed fashion
            params: list[Tensor] = group["params"]
            handle = None
            params_world = None
            def update_prev(): # optimized Muon implementation contributed by @YouJiacheng
                if params_world is None:
                    return
                assert handle is not None
                handle.wait()
                for p_world, g_world in zip(params_world, update_buffer_views):
                    p_world.add_(
                        g_world.view_as(p_world),
                        alpha=-lr * max(1, p_world.size(-2) / p_world.size(-1)) ** 0.5,
                    )
            for base_i in range(len(params))[::self.world_size]:
                if base_i + self.rank < len(params):
                    p = params[base_i + self.rank]
                    g = p.grad
                    assert g is not None
                    state = self.state[p]
                    if "momentum_buffer" not in state:
                        state["momentum_buffer"] = torch.zeros_like(g)
                    buf: Tensor = state["momentum_buffer"]
                    buf.lerp_(g, 1 - momentum)
                    g = g.lerp_(buf, momentum) if nesterov else buf
                    g = zeropower_via_newtonschulz5(g, steps=ns_steps).flatten()
                else:
                    g = update_buffer_views[self.rank]
                update_prev() # async all_gather instead of sync all_reduce by @YouJiacheng
                handle = dist.all_gather_into_tensor(update_buffer, g, async_op=True)
                params_world = params[base_i : base_i + self.world_size]
            update_prev()

# -----------------------------------------------------------------------------
# PyTorch nn.Module definitions for the model

def norm(x: Tensor):
    return F.rms_norm(x, (x.size(-1),))

class CastedLinear(nn.Linear):
    def __init__(self, in_features: int, out_features: int, use_fp8: bool = False, x_s: float = 1.0, w_s: float = 1.0, grad_s: float = 1.0):
        super().__init__(in_features, out_features, bias=False)
        self.use_fp8 = use_fp8
        self.x_s = x_s
        self.w_s = w_s
        self.grad_s = grad_s

    def reset_parameters(self) -> None:
        std = 0.5 * (self.in_features ** -0.5) # 0.5 is a bit better than the default 1/sqrt(3)
        bound = (3 ** 0.5) * std
        with torch.no_grad():
            self.weight.uniform_(-bound, bound)

    def forward(self, x: Tensor):
        if self.use_fp8 and self.training:
            _x = x.flatten(0, -2)
            out: Tensor = torch.ops.nanogpt.mm(_x, self.weight, x_s=self.x_s, w_s=self.w_s, grad_s=self.grad_s)[0]
            return out.reshape(*x.shape[:-1], -1)
        else:
            return F.linear(x, self.weight.type_as(x))

class Rotary(nn.Module):
    def __init__(self, dim: int, max_seq_len: int):
        super().__init__()
        # half-truncate RoPE by @YouJiacheng (w/ base freq tuning)
        angular_freq = (1 / 1024) ** torch.linspace(0, 1, steps=dim//4, dtype=torch.float32)
        angular_freq = torch.cat([angular_freq, angular_freq.new_zeros(dim//4)])
        t = torch.arange(max_seq_len, dtype=torch.float32)
        theta = torch.einsum("i,j -> ij", t, angular_freq)
        self.cos = nn.Buffer(theta.cos(), persistent=False)
        self.sin = nn.Buffer(theta.sin(), persistent=False)

    def forward(self, x_BTHD: Tensor):
        assert self.cos.size(0) >= x_BTHD.size(-3)
        cos, sin = self.cos[None, :x_BTHD.size(-3), None, :], self.sin[None, :x_BTHD.size(-3), None, :]
        x1, x2 = x_BTHD.to(dtype=torch.float32).chunk(2, dim=-1)
        y1 = x1 * cos + x2 * sin
        y2 = x1 * (-sin) + x2 * cos
        return torch.cat((y1, y2), 3).type_as(x_BTHD)

class CausalSelfAttention(nn.Module):
    def __init__(self, dim: int, num_heads: int, max_seq_len: int, head_dim=128):
        super().__init__()
        self.num_heads = num_heads
        self.head_dim = head_dim
        hdim = num_heads * head_dim
        std = 0.5 * (dim ** -0.5)
        bound = (3 ** 0.5) * std # improved init scale by @YouJiacheng
        # merged QKV weights: suggested by many, implemented by @fernbear.bsky.social, and further improved by @YouJiacheng
        # https://x.com/hi_tysam/status/1879699187107033311
        self.qkv_w = nn.Parameter(torch.empty(3, hdim, dim).uniform_(-bound, bound))
        self.lambdas = nn.Parameter(torch.tensor([0.5, 0.5]))
        self.rotary = Rotary(head_dim, max_seq_len)
        self.c_proj = CastedLinear(hdim, dim)
        self.c_proj.weight.detach().zero_() # zero init suggested by @Grad62304977
        # scale the attention logits by given constant, instead of the default head_dim**-0.5, by @leloykun
        # inspired by learnable scalars used by @brendanh0gan https://x.com/hi_tysam/status/1879693583898591283
        self.attn_scale = 0.12

    def forward(self, x: Tensor, ve: Tensor | None, block_mask: BlockMask):
        B, T = x.size(0), x.size(1) # batch size, sequence length
        assert B == 1, "Must use batch size = 1 for FlexAttention"
        q, k, v = F.linear(x, self.qkv_w.flatten(end_dim=1).type_as(x)).view(B, T, 3 * self.num_heads, self.head_dim).chunk(3, dim=-2)
        q, k = norm(q), norm(k) # QK norm @Grad62304977
        q, k = self.rotary(q), self.rotary(k)
        if ve is not None:
            v = self.lambdas[0] * v + self.lambdas[1] * ve.view_as(v) # @KoszarskyB & @Grad62304977
        else: # skip mid-layers token value embeddings by @YouJiacheng
            v = self.lambdas[0] * v
        y = flex_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), block_mask=block_mask, scale=self.attn_scale).transpose(1, 2)
        y = y.contiguous().view(B, T, self.num_heads * self.head_dim) # re-assemble all head outputs side by side
        y = self.c_proj(y)
        return y

class MLP(nn.Module):
    def __init__(self, dim: int):
        super().__init__()
        hdim = 4 * dim
        self.c_fc = CastedLinear(dim, hdim)
        self.c_proj = CastedLinear(hdim, dim)
        self.c_proj.weight.detach().zero_() # zero init suggested by @Grad62304977

    def forward(self, x: Tensor):
        x = self.c_fc(x)
        x = F.relu(x).square() # https://arxiv.org/abs/2109.08668v2; ~1-2% better than GELU; suggested by @SKYLINEZ007 and @Grad62304977
        x = self.c_proj(x)
        return x

class Block(nn.Module):
    def __init__(self, dim: int, num_heads: int, layer_idx: int, max_seq_len: int):
        super().__init__()
        # skip attention of blocks.7 (the 8th layer) by @YouJiacheng
        self.attn = CausalSelfAttention(dim, num_heads, max_seq_len) if layer_idx != 7 else None
        self.mlp = MLP(dim)
        self.lambdas = nn.Parameter(torch.tensor([1., 0.]))

    def forward(self, x: Tensor, ve: Tensor | None, x0: Tensor, block_mask: BlockMask):
        x = self.lambdas[0] * x + self.lambdas[1] * x0
        if self.attn is not None:
            x = x + self.attn(norm(x), ve, block_mask)
        x = x + self.mlp(norm(x))
        return x

class ValueEmbedding(nn.Module):
    def __init__(self, vocab_size: int, embedding_dim: int, num_layers: int, num_embeddings: int = 3):
        super().__init__()
        self.num_layers = num_layers
        self.num_embeddings = num_embeddings
        self.embed = nn.ModuleList([nn.Embedding(vocab_size, embedding_dim) for _ in range(num_embeddings)])

    def forward(self, input_seq: Tensor) -> list[Tensor | None]:
        ve = [emb(input_seq) for emb in self.embed]
        # 012 ... 012 structure on token value embeddings by @YouJiacheng, improved on @leloykun's U-net structure
        ve = [ve[0], ve[1], ve[2]] + [None] * (self.num_layers - 2 * self.num_embeddings) + [ve[0], ve[1], ve[2]]
        return ve

# -----------------------------------------------------------------------------
# The main model

def next_multiple_of_n(v: float | int, *, n: int):
    return next(x for x in range(n, int(v) + 1 + n, n) if x >= v)

class GPT(nn.Module):
    def __init__(self, vocab_size: int, num_layers: int, num_heads: int, model_dim: int, max_seq_len: int):
        super().__init__()
        self.embed = nn.Embedding(vocab_size, model_dim)
        # token value embeddings by @KoszarskyB - inspired by @Grad62304977's value residual implementation following https://arxiv.org/abs/2410.17897
        self.value_embeds = ValueEmbedding(vocab_size, model_dim, num_layers)
        self.blocks = nn.ModuleList([Block(model_dim, num_heads, layer_idx, max_seq_len) for layer_idx in range(num_layers)])
        # U-net design by @brendanh0gan
        self.num_encoder_layers = num_layers // 2 # Half of the layers for encoder
        self.num_decoder_layers = num_layers - self.num_encoder_layers # Remaining for decoder
        # Add learnable skip connection weights for decoder layers
        self.skip_weights = nn.Parameter(torch.ones(self.num_decoder_layers))
        # there are only 50257 unique GPT-2 tokens; we extend to nearest multiple of 128 for efficiency.
        # suggested to me by @Grad62304977. this originates from Karpathy's experiments.
        self.lm_head = CastedLinear(model_dim, next_multiple_of_n(vocab_size, n=128), use_fp8=True, x_s=2.0, w_s=2.0**9, grad_s=2.0**19)
        self.lm_head.weight.detach().zero_() # @Grad62304977

    def create_block_masks(self, input_seq: Tensor, sliding_window_num_blocks: Tensor):
        BLOCK_SIZE = 128
        docs = (input_seq == 50256).cumsum(0)

        def document_causal(b, h, q_idx, kv_idx):
            causal_mask = q_idx >= kv_idx
            document_mask = docs[q_idx] == docs[kv_idx]
            return causal_mask & document_mask

        def dense_to_ordered(dense_mask: Tensor):
            num_blocks = dense_mask.sum(dim=-1, dtype=torch.int32)
            indices = dense_mask.argsort(dim=-1, descending=False, stable=True).flip(-1).to(torch.int32)
            return num_blocks[None, None].contiguous(), indices[None, None].contiguous()

        # manual block mask creation by @YouJiacheng
        assert len(input_seq) % BLOCK_SIZE == 0
        NUM_BLOCKS = len(input_seq) // BLOCK_SIZE
        block_idx = torch.arange(NUM_BLOCKS, dtype=torch.int32, device="cuda")
        any_causal_bm = block_idx[:, None] >= block_idx
        all_causal_bm = block_idx[:, None] > block_idx
        docs_low = docs.view(-1, BLOCK_SIZE)[:, 0].contiguous()
        docs_high = docs.view(-1, BLOCK_SIZE)[:, -1].contiguous()
        any_document_bm = (docs_low[:, None] <= docs_high) & (docs_high[:, None] >= docs_low)
        all_document_bm = (docs_low[:, None] == docs_high) & (docs_high[:, None] == docs_low)
        any_bm = any_causal_bm & any_document_bm
        all_bm = all_causal_bm & all_document_bm
        partial_kv_num_blocks, partial_kv_indices = dense_to_ordered(any_bm & ~all_bm)
        full_kv_num_blocks, full_kv_indices = dense_to_ordered(all_bm)
        def build_bm(sw_num_blocks: Tensor) -> BlockMask:
            return BlockMask.from_kv_blocks(
                torch.clamp_max(partial_kv_num_blocks, torch.clamp_min(sw_num_blocks - full_kv_num_blocks, 1)),
                partial_kv_indices,
                torch.clamp_max(full_kv_num_blocks, sw_num_blocks - 1),
                full_kv_indices,
                BLOCK_SIZE=BLOCK_SIZE,
                mask_mod=document_causal,
            )
        # Long-short SWA block masks by @leloykun & @YouJiacheng, adapated from suggestion by @Grad62304977, following Gemma 2 paper
        return build_bm(sliding_window_num_blocks), build_bm(sliding_window_num_blocks // 2)

    def forward(self, input_seq: Tensor, target_seq: Tensor, sliding_window_num_blocks: Tensor):
        assert input_seq.ndim == 1

        long_bm, short_bm = self.create_block_masks(input_seq, sliding_window_num_blocks)

        x = x0 = norm(self.embed(input_seq)[None]) # use of norm here by @Grad62304977
        ve = self.value_embeds(input_seq)
        assert len(ve) == len(self.blocks)
        ve_enc, ve_dec = ve[:self.num_encoder_layers], ve[self.num_encoder_layers:]
        assert len(ve_enc) == self.num_encoder_layers and len(ve_dec) == self.num_decoder_layers

        # Store outputs for U-Net skip connections
        skip_connections = []
        # Encoder pass - process only the first half of the blocks
        block_masks = [long_bm, short_bm, short_bm, short_bm, long_bm, short_bm]
        assert len(block_masks) == self.num_encoder_layers
        for i in range(self.num_encoder_layers):
            x = self.blocks[i](x, ve_enc[i], x0, block_masks[i])
            skip_connections.append(x)
        # Decoder pass - process the remaining blocks with weighted skip connections
        block_masks.reverse()
        assert len(block_masks) == self.num_decoder_layers
        for i in range(self.num_decoder_layers):
            x = x + self.skip_weights[i] * skip_connections.pop()
            x = self.blocks[self.num_encoder_layers + i](x, ve_dec[i], x0, block_masks[i])
        x = norm(x)
        logits = self.lm_head(x)
        # @Grad62304977 added tanh softcapping following Gemma 2 paper, @KoszarskyB reduced it from 30 to 15, @YouJiacheng shifted it by +15 (2*sigmoid(2*x)=tanh(x)+1)
        logits = 30 * torch.sigmoid(logits.float() / 7.5)
        loss = F.cross_entropy(logits.view(-1, logits.size(-1)), target_seq)
        return loss

# -----------------------------------------------------------------------------
# Our own simple Distributed Data Loader

def _load_data_shard(file: Path):
    header = torch.from_file(f"{file}", False, 256, dtype=torch.int32) # header is 256 int32
    assert header[0] == 20240520, "magic number mismatch in the data .bin file"
    assert header[1] == 1, "unsupported version"
    num_tokens = int(header[2]) # number of tokens (claimed)
    with file.open("rb", buffering=0) as f:
        tokens = torch.empty(num_tokens, dtype=torch.uint16, pin_memory=True) # avoid pin_memory copy by @YouJiacheng
        f.seek(256 * 4)
        nbytes = f.readinto(tokens.numpy()) # avoid bytes->array copy by @YouJiacheng
        assert nbytes == 2 * num_tokens, "number of tokens read does not match header"
    return tokens

def distributed_data_generator(filename_pattern: str, batch_size: int, rank : int, world_size : int):
    files = sorted(Path.cwd().glob(filename_pattern))
    assert batch_size % world_size == 0
    local_batch_size = batch_size // world_size
    file_iter = iter(files) # use itertools.cycle(files) instead if you want to do multi-epoch training
    tokens, pos = _load_data_shard(next(file_iter)), 0
    while True:
        if pos + batch_size + 1 >= len(tokens):
            tokens, pos = _load_data_shard(next(file_iter)), 0
        buf = tokens[pos + rank * local_batch_size:][:local_batch_size + 1]
        inputs = buf[:-1].to(device="cuda", dtype=torch.int32, non_blocking=True) # no sync on host side;
        targets = buf[1:].to(device="cuda", dtype=torch.int64, non_blocking=True) # H2D in another stream isn"t helpful.
        pos += batch_size
        yield inputs, targets

# -----------------------------------------------------------------------------
# int main

@dataclass
class Hyperparameters:
    # data
    train_files = "data/fineweb10B/fineweb_train_*.bin" # input .bin to train on
    val_files = "data/fineweb10B/fineweb_val_*.bin" # input .bin to eval validation loss on
    val_tokens = 10485760 # how many tokens of validation data? it's important to keep this fixed for consistent comparisons
    # optimization
    num_iterations = 1770 # number of iterations to run
    cooldown_frac = 0.4 # fraction of training spent cooling down the learning rate
    # evaluation and logging
    val_loss_every = 125 # every how many steps to evaluate val loss? 0 for only at the end
    # implementation
    seq_len = 48*1024 # FlexAttention sequence length
    val_seq_len = 4*64*1024 # FlexAttention sequence length for validation
    save_checkpoint = False
args = Hyperparameters()

# torchrun sets these env variables
rank = int(os.environ["RANK"])
world_size = int(os.environ["WORLD_SIZE"])
assert torch.cuda.is_available()
device = torch.device("cuda", int(os.environ["LOCAL_RANK"]))
torch.cuda.set_device(device)
dist.init_process_group(backend="nccl", device_id=device)
dist.barrier()
master_process = (rank == 0) # this process will do logging, checkpointing etc.

# begin logging
logfile = None
if master_process:
    run_id = uuid.uuid4()
    os.makedirs("logs", exist_ok=True)
    logfile = f"logs/{run_id}.txt"
    print(logfile)
def print0(s, console=False):
    if master_process:
        with open(logfile, "a") as f:
            if console:
                print(s)
            print(s, file=f)

# begin by printing this file (the Python code)
print0(code)
print0("="*100)
# log information about the hardware/software environment this is running on
print0(f"Running Python {sys.version}")
print0(f"Running PyTorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}")
def nvidia_smi():
    import subprocess  # avoid top level import
    return subprocess.run(["nvidia-smi"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True).stdout
print0(nvidia_smi())
print0("="*100)

# load data
train_batch_size = world_size * args.seq_len
train_loader = distributed_data_generator(args.train_files, train_batch_size, rank, world_size)

model: nn.Module = GPT(vocab_size=50257, num_layers=12, num_heads=6, model_dim=768, max_seq_len=max(args.seq_len, args.val_seq_len)).cuda()
for m in model.modules():
    if isinstance(m, nn.Embedding):
        m.bfloat16()
for param in model.parameters():
    dist.broadcast(param.detach(), 0)

# collect the parameters to optimize
hidden_matrix_params = [p for n, p in model.blocks.named_parameters() if p.ndim >= 2 and "embed" not in n]
embed_params = [p for n, p in model.named_parameters() if "embed" in n]
scalar_params = [p for p in model.parameters() if p.ndim < 2]
head_params = [model.lm_head.weight]

# init the optimizer(s)
adam_params = [dict(params=head_params, lr=0.008), dict(params=embed_params, lr=0.6), dict(params=scalar_params, lr=0.04)]
# small adam epsilon by @YouJiacheng. this is an alternate method of fixing the world_size dependence
# discovered by @fernbear.bsky.social https://x.com/hi_tysam/status/1879692937589875094
optimizer1 = torch.optim.Adam(adam_params, betas=(0.8, 0.95), eps=1e-10, fused=True)
optimizer2 = Muon(hidden_matrix_params, lr=0.05, momentum=0.95, rank=rank, world_size=world_size)
optimizers = [optimizer1, optimizer2]

# learning rate schedule: stable then decay
def get_lr(step: int):
    t = 1 - step / args.num_iterations # time remaining in training
    assert 1 >= t >= 0
    w = min(t / args.cooldown_frac, 1.0) # 1 -> 0
    return w * 1.0 + (1 - w) * 0.1
schedulers = [torch.optim.lr_scheduler.LambdaLR(opt, get_lr) for opt in optimizers]
@lru_cache(1)
def sw_num_blks(window_size: int):
    return torch.tensor(window_size // 128, dtype=torch.int32, pin_memory=True).cuda(non_blocking=True)

model: nn.Module = torch.compile(model, dynamic=False)

training_time_ms = 0
# start the clock
torch.cuda.synchronize()
t0 = time.perf_counter()
# begin training
train_steps = args.num_iterations
for step in range(train_steps + 1):
    last_step = (step == train_steps)
    # This effectively ignores timing first 10 steps, which are slower for weird reasons.
    # Alternately, and slightly more correctly in terms of benchmarking, we could do 10
    # steps with dummy data first, and then re-initialize the model and reset the loader.
    if step == 10:
        training_time_ms = 0
        t0 = time.perf_counter()
    timed_steps = float("nan") if step <= 11 else (step - 10) + 1 # <= 11 to avoid bug in val

    # Linearly increase the block-wise sliding window size over training 128 -> 1792:
    # increase by @fernbear.bsky.social; block-wise by @YouJiacheng
    window_size = next_multiple_of_n(1728 * step / train_steps, n=128)

    # --------------- VALIDATION SECTION -----------------
    if last_step or (args.val_loss_every > 0 and step % args.val_loss_every == 0):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        model.eval()
        val_batch_size = world_size * args.val_seq_len
        assert args.val_tokens % val_batch_size == 0
        val_steps = args.val_tokens // val_batch_size
        val_loader = distributed_data_generator(args.val_files, val_batch_size, rank, world_size)
        val_loss = 0
        with torch.no_grad():
            for _ in range(val_steps):
                x, y = next(val_loader)
                val_loss += model(x, y, sw_num_blks(window_size))
        val_loss /= val_steps
        del val_loader
        dist.all_reduce(val_loss, op=dist.ReduceOp.AVG)
        print0(f"step:{step}/{train_steps} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms", console=True)
        model.train()
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    if last_step:
        if master_process and args.save_checkpoint:
            log = dict(step=step, code=code, model=model.state_dict(), optimizers=[opt.state_dict() for opt in optimizers])
            os.makedirs(f"logs/{run_id}", exist_ok=True)
            torch.save(log, f"logs/{run_id}/state_step{step:06d}.pt")
        # the last step only has the validation loop, so break to avoid training
        break

    # --------------- TRAINING SECTION -----------------
    inputs, targets = next(train_loader)
    for input_seq, target_seq in zip(inputs.split(args.seq_len), targets.split(args.seq_len)):
        model(input_seq, target_seq, sw_num_blks(window_size)).backward()
    for param in model.parameters():
        dist.all_reduce(param.grad, op=dist.ReduceOp.AVG)
    # momentum warmup for Muon
    frac = min(step / 300, 1)
    for group in optimizer2.param_groups:
        group["momentum"] = (1 - frac) * 0.85 + frac * 0.95
    # step the optimizers and schedulers
    for opt, sched in zip(optimizers, schedulers):
        opt.step()
        sched.step()
    # null the gradients
    model.zero_grad(set_to_none=True)
    # logging
    approx_time = training_time_ms + 1000 * (time.perf_counter() - t0)
    print0(f"step:{step+1}/{train_steps} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms", console=True)

print0(
    f"peak memory allocated: {torch.cuda.max_memory_allocated() // 1024 // 1024} MiB "
    f"reserved: {torch.cuda.max_memory_reserved() // 1024 // 1024} MiB",
    console=True,
)
dist.destroy_process_group()

====================================================================================================
Running Python 3.12.7 (main, Feb  1 2025, 03:09:49) [GCC 13.2.0]
Running PyTorch 2.7.0.dev20250125+cu126 compiled for CUDA 12.6
Sat Feb  1 20:42:40 2025       
+-----------------------------------------------------------------------------------------+
| NVIDIA-SMI 550.127.08             Driver Version: 550.127.08     CUDA Version: 12.6     |
|-----------------------------------------+------------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id          Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |           Memory-Usage | GPU-Util  Compute M. |
|                                         |                        |               MIG M. |
|=========================================+========================+======================|
|   0  NVIDIA H100 80GB HBM3          On  |   00000000:61:00.0 Off |                    0 |
| N/A   27C    P0            122W /  700W |    7746MiB /  81559MiB |      0%      Default |
|                                         |                        |             Disabled |
+-----------------------------------------+------------------------+----------------------+
|   1  NVIDIA H100 80GB HBM3          On  |   00000000:62:00.0 Off |                    0 |
| N/A   31C    P0            119W /  700W |    3456MiB /  81559MiB |      0%      Default |
|                                         |                        |             Disabled |
+-----------------------------------------+------------------------+----------------------+
|   2  NVIDIA H100 80GB HBM3          On  |   00000000:63:00.0 Off |                    0 |
| N/A   31C    P0            117W /  700W |    3456MiB /  81559MiB |      0%      Default |
|                                         |                        |             Disabled |
+-----------------------------------------+------------------------+----------------------+
|   3  NVIDIA H100 80GB HBM3          On  |   00000000:64:00.0 Off |                    0 |
| N/A   27C    P0            114W /  700W |    3456MiB /  81559MiB |      0%      Default |
|                                         |                        |             Disabled |
+-----------------------------------------+------------------------+----------------------+
|   4  NVIDIA H100 80GB HBM3          On  |   00000000:6A:00.0 Off |                    0 |
| N/A   27C    P0            125W /  700W |    3456MiB /  81559MiB |      0%      Default |
|                                         |                        |             Disabled |
+-----------------------------------------+------------------------+----------------------+
|   5  NVIDIA H100 80GB HBM3          On  |   00000000:6B:00.0 Off |                    0 |
| N/A   31C    P0            120W /  700W |    3456MiB /  81559MiB |      0%      Default |
|                                         |                        |             Disabled |
+-----------------------------------------+------------------------+----------------------+
|   6  NVIDIA H100 80GB HBM3          On  |   00000000:6C:00.0 Off |                    0 |
| N/A   29C    P0            120W /  700W |    3456MiB /  81559MiB |      0%      Default |
|                                         |                        |             Disabled |
+-----------------------------------------+------------------------+----------------------+
|   7  NVIDIA H100 80GB HBM3          On  |   00000000:6D:00.0 Off |                    0 |
| N/A   26C    P0            119W /  700W |    3216MiB /  81559MiB |      0%      Default |
|                                         |                        |             Disabled |
+-----------------------------------------+------------------------+----------------------+
                                                                                         
+-----------------------------------------------------------------------------------------+
| Processes:                                                                              |
|  GPU   GI   CI        PID   Type   Process name                              GPU Memory |
|        ID   ID                                                               Usage      |
|=========================================================================================|
+-----------------------------------------------------------------------------------------+

====================================================================================================
step:0/1770 val_loss:10.8258 train_time:0ms step_avg:nanms
step:1/1770 train_time:22693ms step_avg:nanms
step:2/1770 train_time:23075ms step_avg:nanms
step:3/1770 train_time:23169ms step_avg:nanms
step:4/1770 train_time:23261ms step_avg:nanms
step:5/1770 train_time:23356ms step_avg:nanms
step:6/1770 train_time:23451ms step_avg:nanms
step:7/1770 train_time:23545ms step_avg:nanms
step:8/1770 train_time:23639ms step_avg:nanms
step:9/1770 train_time:23733ms step_avg:nanms
step:10/1770 train_time:23827ms step_avg:nanms
step:11/1770 train_time:94ms step_avg:nanms
step:12/1770 train_time:188ms step_avg:nanms
step:13/1770 train_time:283ms step_avg:94.48ms
step:14/1770 train_time:379ms step_avg:94.63ms
step:15/1770 train_time:473ms step_avg:94.56ms
step:16/1770 train_time:567ms step_avg:94.51ms
step:17/1770 train_time:661ms step_avg:94.49ms
step:18/1770 train_time:755ms step_avg:94.39ms
step:19/1770 train_time:849ms step_avg:94.31ms
step:20/1770 train_time:943ms step_avg:94.34ms
step:21/1770 train_time:1037ms step_avg:94.31ms
step:22/1770 train_time:1131ms step_avg:94.26ms
step:23/1770 train_time:1225ms step_avg:94.26ms
step:24/1770 train_time:1321ms step_avg:94.33ms
step:25/1770 train_time:1415ms step_avg:94.30ms
step:26/1770 train_time:1509ms step_avg:94.29ms
step:27/1770 train_time:1603ms step_avg:94.29ms
step:28/1770 train_time:1697ms step_avg:94.28ms
step:29/1770 train_time:1791ms step_avg:94.27ms
step:30/1770 train_time:1886ms step_avg:94.28ms
step:31/1770 train_time:1980ms step_avg:94.27ms
step:32/1770 train_time:2074ms step_avg:94.27ms
step:33/1770 train_time:2168ms step_avg:94.24ms
step:34/1770 train_time:2262ms step_avg:94.25ms
step:35/1770 train_time:2357ms step_avg:94.27ms
step:36/1770 train_time:2451ms step_avg:94.26ms
step:37/1770 train_time:2545ms step_avg:94.26ms
step:38/1770 train_time:2640ms step_avg:94.27ms
step:39/1770 train_time:2734ms step_avg:94.28ms
step:40/1770 train_time:2828ms step_avg:94.28ms
step:41/1770 train_time:2923ms step_avg:94.28ms
step:42/1770 train_time:3018ms step_avg:94.31ms
step:43/1770 train_time:3112ms step_avg:94.31ms
step:44/1770 train_time:3208ms step_avg:94.36ms
step:45/1770 train_time:3301ms step_avg:94.32ms
step:46/1770 train_time:3397ms step_avg:94.35ms
step:47/1770 train_time:3491ms step_avg:94.34ms
step:48/1770 train_time:3586ms step_avg:94.37ms
step:49/1770 train_time:3680ms step_avg:94.37ms
step:50/1770 train_time:3775ms step_avg:94.37ms
step:51/1770 train_time:3869ms step_avg:94.36ms
step:52/1770 train_time:3963ms step_avg:94.37ms
step:53/1770 train_time:4058ms step_avg:94.37ms
step:54/1770 train_time:4152ms step_avg:94.36ms
step:55/1770 train_time:4246ms step_avg:94.36ms
step:56/1770 train_time:4341ms step_avg:94.37ms
step:57/1770 train_time:4436ms step_avg:94.39ms
step:58/1770 train_time:4531ms step_avg:94.39ms
step:59/1770 train_time:4625ms step_avg:94.40ms
step:60/1770 train_time:4720ms step_avg:94.40ms
step:61/1770 train_time:4815ms step_avg:94.42ms
step:62/1770 train_time:4909ms step_avg:94.40ms
step:63/1770 train_time:5003ms step_avg:94.40ms
step:64/1770 train_time:5098ms step_avg:94.40ms
step:65/1770 train_time:5192ms step_avg:94.40ms
step:66/1770 train_time:5286ms step_avg:94.39ms
step:67/1770 train_time:5380ms step_avg:94.39ms
step:68/1770 train_time:5475ms step_avg:94.40ms
step:69/1770 train_time:5569ms step_avg:94.39ms
step:70/1770 train_time:5664ms step_avg:94.39ms
step:71/1770 train_time:5758ms step_avg:94.40ms
step:72/1770 train_time:5853ms step_avg:94.40ms
step:73/1770 train_time:5946ms step_avg:94.39ms
step:74/1770 train_time:6041ms step_avg:94.39ms
step:75/1770 train_time:6135ms step_avg:94.39ms
step:76/1770 train_time:6229ms step_avg:94.38ms
step:77/1770 train_time:6324ms step_avg:94.39ms
step:78/1770 train_time:6419ms step_avg:94.39ms
step:79/1770 train_time:6513ms step_avg:94.39ms
step:80/1770 train_time:6607ms step_avg:94.39ms
step:81/1770 train_time:6702ms step_avg:94.39ms
step:82/1770 train_time:6796ms step_avg:94.39ms
step:83/1770 train_time:6890ms step_avg:94.39ms
step:84/1770 train_time:6984ms step_avg:94.38ms
step:85/1770 train_time:7079ms step_avg:94.39ms
step:86/1770 train_time:7173ms step_avg:94.39ms
step:87/1770 train_time:7268ms step_avg:94.38ms
step:88/1770 train_time:7362ms step_avg:94.38ms
step:89/1770 train_time:7456ms step_avg:94.38ms
step:90/1770 train_time:7550ms step_avg:94.37ms
step:91/1770 train_time:7644ms step_avg:94.37ms
step:92/1770 train_time:7738ms step_avg:94.37ms
step:93/1770 train_time:7832ms step_avg:94.37ms
step:94/1770 train_time:7926ms step_avg:94.36ms
step:95/1770 train_time:8021ms step_avg:94.36ms
step:96/1770 train_time:8115ms step_avg:94.36ms
step:97/1770 train_time:8210ms step_avg:94.37ms
step:98/1770 train_time:8303ms step_avg:94.35ms
step:99/1770 train_time:8398ms step_avg:94.36ms
step:100/1770 train_time:8492ms step_avg:94.36ms
step:101/1770 train_time:8587ms step_avg:94.37ms
step:102/1770 train_time:8682ms step_avg:94.37ms
step:103/1770 train_time:8776ms step_avg:94.37ms
step:104/1770 train_time:8870ms step_avg:94.36ms
step:105/1770 train_time:8965ms step_avg:94.37ms
step:106/1770 train_time:9059ms step_avg:94.36ms
step:107/1770 train_time:9153ms step_avg:94.37ms
step:108/1770 train_time:9248ms step_avg:94.36ms
step:109/1770 train_time:9343ms step_avg:94.37ms
step:110/1770 train_time:9437ms step_avg:94.37ms
step:111/1770 train_time:9531ms step_avg:94.37ms
step:112/1770 train_time:9626ms step_avg:94.37ms
step:113/1770 train_time:9721ms step_avg:94.38ms
step:114/1770 train_time:9815ms step_avg:94.38ms
step:115/1770 train_time:9909ms step_avg:94.37ms
step:116/1770 train_time:10003ms step_avg:94.36ms
step:117/1770 train_time:10098ms step_avg:94.37ms
step:118/1770 train_time:10192ms step_avg:94.37ms
step:119/1770 train_time:10287ms step_avg:94.38ms
step:120/1770 train_time:10382ms step_avg:94.38ms
step:121/1770 train_time:10478ms step_avg:94.39ms
step:122/1770 train_time:10572ms step_avg:94.39ms
step:123/1770 train_time:10666ms step_avg:94.39ms
step:124/1770 train_time:10761ms step_avg:94.39ms
step:125/1770 train_time:10855ms step_avg:94.39ms
step:125/1770 val_loss:4.6549 train_time:10947ms step_avg:95.19ms
step:126/1770 train_time:10970ms step_avg:94.57ms
step:127/1770 train_time:11045ms step_avg:94.40ms
step:128/1770 train_time:11147ms step_avg:94.47ms
step:129/1770 train_time:11250ms step_avg:94.54ms
step:130/1770 train_time:11346ms step_avg:94.55ms
step:131/1770 train_time:11440ms step_avg:94.55ms
step:132/1770 train_time:11535ms step_avg:94.55ms
step:133/1770 train_time:11629ms step_avg:94.55ms
step:134/1770 train_time:11723ms step_avg:94.54ms
step:135/1770 train_time:11817ms step_avg:94.54ms
step:136/1770 train_time:11912ms step_avg:94.54ms
step:137/1770 train_time:12007ms step_avg:94.54ms
step:138/1770 train_time:12102ms step_avg:94.54ms
step:139/1770 train_time:12197ms step_avg:94.55ms
step:140/1770 train_time:12292ms step_avg:94.56ms
step:141/1770 train_time:12388ms step_avg:94.56ms
step:142/1770 train_time:12482ms step_avg:94.56ms
step:143/1770 train_time:12578ms step_avg:94.57ms
step:144/1770 train_time:12672ms step_avg:94.57ms
step:145/1770 train_time:12767ms step_avg:94.57ms
step:146/1770 train_time:12862ms step_avg:94.57ms
step:147/1770 train_time:12957ms step_avg:94.58ms
step:148/1770 train_time:13052ms step_avg:94.58ms
step:149/1770 train_time:13147ms step_avg:94.58ms
step:150/1770 train_time:13243ms step_avg:94.59ms
step:151/1770 train_time:13338ms step_avg:94.60ms
step:152/1770 train_time:13432ms step_avg:94.59ms
step:153/1770 train_time:13527ms step_avg:94.60ms
step:154/1770 train_time:13624ms step_avg:94.61ms
step:155/1770 train_time:13718ms step_avg:94.61ms
step:156/1770 train_time:13813ms step_avg:94.61ms
step:157/1770 train_time:13908ms step_avg:94.61ms
step:158/1770 train_time:14004ms step_avg:94.62ms
step:159/1770 train_time:14098ms step_avg:94.62ms
step:160/1770 train_time:14192ms step_avg:94.62ms
step:161/1770 train_time:14288ms step_avg:94.62ms
step:162/1770 train_time:14382ms step_avg:94.62ms
step:163/1770 train_time:14477ms step_avg:94.62ms
step:164/1770 train_time:14572ms step_avg:94.62ms
step:165/1770 train_time:14667ms step_avg:94.63ms
step:166/1770 train_time:14762ms step_avg:94.63ms
step:167/1770 train_time:14856ms step_avg:94.63ms
step:168/1770 train_time:14951ms step_avg:94.63ms
step:169/1770 train_time:15046ms step_avg:94.63ms
step:170/1770 train_time:15142ms step_avg:94.63ms
step:171/1770 train_time:15237ms step_avg:94.64ms
step:172/1770 train_time:15331ms step_avg:94.64ms
step:173/1770 train_time:15426ms step_avg:94.64ms
step:174/1770 train_time:15521ms step_avg:94.64ms
step:175/1770 train_time:15616ms step_avg:94.65ms
step:176/1770 train_time:15711ms step_avg:94.65ms
step:177/1770 train_time:15806ms step_avg:94.65ms
step:178/1770 train_time:15901ms step_avg:94.65ms
step:179/1770 train_time:15996ms step_avg:94.65ms
step:180/1770 train_time:16091ms step_avg:94.66ms
step:181/1770 train_time:16187ms step_avg:94.66ms
step:182/1770 train_time:16282ms step_avg:94.66ms
step:183/1770 train_time:16376ms step_avg:94.66ms
step:184/1770 train_time:16472ms step_avg:94.67ms
step:185/1770 train_time:16567ms step_avg:94.67ms
step:186/1770 train_time:16662ms step_avg:94.67ms
step:187/1770 train_time:16758ms step_avg:94.68ms
step:188/1770 train_time:16852ms step_avg:94.67ms
step:189/1770 train_time:16947ms step_avg:94.68ms
step:190/1770 train_time:17042ms step_avg:94.68ms
step:191/1770 train_time:17137ms step_avg:94.68ms
step:192/1770 train_time:17231ms step_avg:94.68ms
step:193/1770 train_time:17326ms step_avg:94.68ms
step:194/1770 train_time:17421ms step_avg:94.68ms
step:195/1770 train_time:17516ms step_avg:94.68ms
step:196/1770 train_time:17611ms step_avg:94.68ms
step:197/1770 train_time:17706ms step_avg:94.69ms
step:198/1770 train_time:17802ms step_avg:94.69ms
step:199/1770 train_time:17897ms step_avg:94.69ms
step:200/1770 train_time:17992ms step_avg:94.69ms
step:201/1770 train_time:18086ms step_avg:94.69ms
step:202/1770 train_time:18182ms step_avg:94.70ms
step:203/1770 train_time:18277ms step_avg:94.70ms
step:204/1770 train_time:18372ms step_avg:94.70ms
step:205/1770 train_time:18467ms step_avg:94.70ms
step:206/1770 train_time:18561ms step_avg:94.70ms
step:207/1770 train_time:18656ms step_avg:94.70ms
step:208/1770 train_time:18752ms step_avg:94.70ms
step:209/1770 train_time:18846ms step_avg:94.71ms
step:210/1770 train_time:18942ms step_avg:94.71ms
step:211/1770 train_time:19037ms step_avg:94.71ms
step:212/1770 train_time:19131ms step_avg:94.71ms
step:213/1770 train_time:19226ms step_avg:94.71ms
step:214/1770 train_time:19321ms step_avg:94.71ms
step:215/1770 train_time:19416ms step_avg:94.71ms
step:216/1770 train_time:19511ms step_avg:94.71ms
step:217/1770 train_time:19605ms step_avg:94.71ms
step:218/1770 train_time:19701ms step_avg:94.72ms
step:219/1770 train_time:19795ms step_avg:94.71ms
step:220/1770 train_time:19890ms step_avg:94.71ms
step:221/1770 train_time:19985ms step_avg:94.72ms
step:222/1770 train_time:20080ms step_avg:94.72ms
step:223/1770 train_time:20176ms step_avg:94.72ms
step:224/1770 train_time:20271ms step_avg:94.72ms
step:225/1770 train_time:20366ms step_avg:94.73ms
step:226/1770 train_time:20462ms step_avg:94.73ms
step:227/1770 train_time:20556ms step_avg:94.73ms
step:228/1770 train_time:20651ms step_avg:94.73ms
step:229/1770 train_time:20745ms step_avg:94.73ms
step:230/1770 train_time:20840ms step_avg:94.73ms
step:231/1770 train_time:20935ms step_avg:94.73ms
step:232/1770 train_time:21031ms step_avg:94.73ms
step:233/1770 train_time:21125ms step_avg:94.73ms
step:234/1770 train_time:21221ms step_avg:94.74ms
step:235/1770 train_time:21316ms step_avg:94.74ms
step:236/1770 train_time:21411ms step_avg:94.74ms
step:237/1770 train_time:21506ms step_avg:94.74ms
step:238/1770 train_time:21601ms step_avg:94.74ms
step:239/1770 train_time:21696ms step_avg:94.74ms
step:240/1770 train_time:21790ms step_avg:94.74ms
step:241/1770 train_time:21885ms step_avg:94.74ms
step:242/1770 train_time:21982ms step_avg:94.75ms
step:243/1770 train_time:22077ms step_avg:94.75ms
step:244/1770 train_time:22171ms step_avg:94.75ms
step:245/1770 train_time:22266ms step_avg:94.75ms
step:246/1770 train_time:22362ms step_avg:94.75ms
step:247/1770 train_time:22457ms step_avg:94.76ms
step:248/1770 train_time:22552ms step_avg:94.76ms
step:249/1770 train_time:22647ms step_avg:94.76ms
step:250/1770 train_time:22742ms step_avg:94.76ms
step:250/1770 val_loss:4.1014 train_time:22835ms step_avg:95.15ms
step:251/1770 train_time:22857ms step_avg:94.84ms
step:252/1770 train_time:22940ms step_avg:94.79ms
step:253/1770 train_time:23040ms step_avg:94.81ms
step:254/1770 train_time:23135ms step_avg:94.82ms
step:255/1770 train_time:23230ms step_avg:94.82ms
step:256/1770 train_time:23324ms step_avg:94.81ms
step:257/1770 train_time:23419ms step_avg:94.81ms
step:258/1770 train_time:23513ms step_avg:94.81ms
step:259/1770 train_time:23608ms step_avg:94.81ms
step:260/1770 train_time:23702ms step_avg:94.81ms
step:261/1770 train_time:23796ms step_avg:94.81ms
step:262/1770 train_time:23891ms step_avg:94.81ms
step:263/1770 train_time:23987ms step_avg:94.81ms
step:264/1770 train_time:24082ms step_avg:94.81ms
step:265/1770 train_time:24177ms step_avg:94.81ms
step:266/1770 train_time:24273ms step_avg:94.82ms
step:267/1770 train_time:24369ms step_avg:94.82ms
step:268/1770 train_time:24463ms step_avg:94.82ms
step:269/1770 train_time:24559ms step_avg:94.82ms
step:270/1770 train_time:24654ms step_avg:94.82ms
step:271/1770 train_time:24749ms step_avg:94.82ms
step:272/1770 train_time:24844ms step_avg:94.82ms
step:273/1770 train_time:24940ms step_avg:94.83ms
step:274/1770 train_time:25035ms step_avg:94.83ms
step:275/1770 train_time:25132ms step_avg:94.84ms
step:276/1770 train_time:25227ms step_avg:94.84ms
step:277/1770 train_time:25322ms step_avg:94.84ms
step:278/1770 train_time:25417ms step_avg:94.84ms
step:279/1770 train_time:25513ms step_avg:94.84ms
step:280/1770 train_time:25608ms step_avg:94.85ms
step:281/1770 train_time:25703ms step_avg:94.85ms
step:282/1770 train_time:25798ms step_avg:94.85ms
step:283/1770 train_time:25895ms step_avg:94.85ms
step:284/1770 train_time:25990ms step_avg:94.85ms
step:285/1770 train_time:26085ms step_avg:94.86ms
step:286/1770 train_time:26181ms step_avg:94.86ms
step:287/1770 train_time:26276ms step_avg:94.86ms
step:288/1770 train_time:26372ms step_avg:94.86ms
step:289/1770 train_time:26467ms step_avg:94.86ms
step:290/1770 train_time:26562ms step_avg:94.86ms
step:291/1770 train_time:26657ms step_avg:94.86ms
step:292/1770 train_time:26752ms step_avg:94.87ms
step:293/1770 train_time:26848ms step_avg:94.87ms
step:294/1770 train_time:26944ms step_avg:94.87ms
step:295/1770 train_time:27039ms step_avg:94.88ms
step:296/1770 train_time:27135ms step_avg:94.88ms
step:297/1770 train_time:27231ms step_avg:94.88ms
step:298/1770 train_time:27327ms step_avg:94.88ms
step:299/1770 train_time:27422ms step_avg:94.89ms
step:300/1770 train_time:27517ms step_avg:94.89ms
step:301/1770 train_time:27612ms step_avg:94.89ms
step:302/1770 train_time:27708ms step_avg:94.89ms
step:303/1770 train_time:27803ms step_avg:94.89ms
step:304/1770 train_time:27899ms step_avg:94.89ms
step:305/1770 train_time:27994ms step_avg:94.90ms
step:306/1770 train_time:28090ms step_avg:94.90ms
step:307/1770 train_time:28185ms step_avg:94.90ms
step:308/1770 train_time:28280ms step_avg:94.90ms
step:309/1770 train_time:28375ms step_avg:94.90ms
step:310/1770 train_time:28471ms step_avg:94.90ms
step:311/1770 train_time:28565ms step_avg:94.90ms
step:312/1770 train_time:28661ms step_avg:94.90ms
step:313/1770 train_time:28756ms step_avg:94.90ms
step:314/1770 train_time:28852ms step_avg:94.91ms
step:315/1770 train_time:28946ms step_avg:94.90ms
step:316/1770 train_time:29042ms step_avg:94.91ms
step:317/1770 train_time:29137ms step_avg:94.91ms
step:318/1770 train_time:29234ms step_avg:94.91ms
step:319/1770 train_time:29329ms step_avg:94.92ms
step:320/1770 train_time:29425ms step_avg:94.92ms
step:321/1770 train_time:29520ms step_avg:94.92ms
step:322/1770 train_time:29615ms step_avg:94.92ms
step:323/1770 train_time:29711ms step_avg:94.92ms
step:324/1770 train_time:29807ms step_avg:94.93ms
step:325/1770 train_time:29902ms step_avg:94.93ms
step:326/1770 train_time:29998ms step_avg:94.93ms
step:327/1770 train_time:30094ms step_avg:94.93ms
step:328/1770 train_time:30189ms step_avg:94.93ms
step:329/1770 train_time:30285ms step_avg:94.94ms
step:330/1770 train_time:30380ms step_avg:94.94ms
step:331/1770 train_time:30476ms step_avg:94.94ms
step:332/1770 train_time:30572ms step_avg:94.94ms
step:333/1770 train_time:30667ms step_avg:94.94ms
step:334/1770 train_time:30762ms step_avg:94.94ms
step:335/1770 train_time:30858ms step_avg:94.95ms
step:336/1770 train_time:30953ms step_avg:94.95ms
step:337/1770 train_time:31047ms step_avg:94.95ms
step:338/1770 train_time:31143ms step_avg:94.95ms
step:339/1770 train_time:31238ms step_avg:94.95ms
step:340/1770 train_time:31334ms step_avg:94.95ms
step:341/1770 train_time:31429ms step_avg:94.95ms
step:342/1770 train_time:31525ms step_avg:94.95ms
step:343/1770 train_time:31620ms step_avg:94.95ms
step:344/1770 train_time:31716ms step_avg:94.96ms
step:345/1770 train_time:31811ms step_avg:94.96ms
step:346/1770 train_time:31906ms step_avg:94.96ms
step:347/1770 train_time:32002ms step_avg:94.96ms
step:348/1770 train_time:32097ms step_avg:94.96ms
step:349/1770 train_time:32192ms step_avg:94.96ms
step:350/1770 train_time:32288ms step_avg:94.96ms
step:351/1770 train_time:32383ms step_avg:94.96ms
step:352/1770 train_time:32478ms step_avg:94.97ms
step:353/1770 train_time:32574ms step_avg:94.97ms
step:354/1770 train_time:32670ms step_avg:94.97ms
step:355/1770 train_time:32765ms step_avg:94.97ms
step:356/1770 train_time:32861ms step_avg:94.97ms
step:357/1770 train_time:32956ms step_avg:94.98ms
step:358/1770 train_time:33052ms step_avg:94.98ms
step:359/1770 train_time:33147ms step_avg:94.98ms
step:360/1770 train_time:33242ms step_avg:94.98ms
step:361/1770 train_time:33337ms step_avg:94.98ms
step:362/1770 train_time:33433ms step_avg:94.98ms
step:363/1770 train_time:33529ms step_avg:94.98ms
step:364/1770 train_time:33624ms step_avg:94.98ms
step:365/1770 train_time:33719ms step_avg:94.98ms
step:366/1770 train_time:33815ms step_avg:94.99ms
step:367/1770 train_time:33911ms step_avg:94.99ms
step:368/1770 train_time:34007ms step_avg:94.99ms
step:369/1770 train_time:34102ms step_avg:94.99ms
step:370/1770 train_time:34197ms step_avg:94.99ms
step:371/1770 train_time:34293ms step_avg:94.99ms
step:372/1770 train_time:34388ms step_avg:95.00ms
step:373/1770 train_time:34483ms step_avg:95.00ms
step:374/1770 train_time:34580ms step_avg:95.00ms
step:375/1770 train_time:34675ms step_avg:95.00ms
step:375/1770 val_loss:3.9027 train_time:34769ms step_avg:95.26ms
step:376/1770 train_time:34791ms step_avg:95.06ms
step:377/1770 train_time:34874ms step_avg:95.02ms
step:378/1770 train_time:34975ms step_avg:95.04ms
step:379/1770 train_time:35070ms step_avg:95.04ms
step:380/1770 train_time:35165ms step_avg:95.04ms
step:381/1770 train_time:35260ms step_avg:95.04ms
step:382/1770 train_time:35355ms step_avg:95.04ms
step:383/1770 train_time:35450ms step_avg:95.04ms
step:384/1770 train_time:35545ms step_avg:95.04ms
step:385/1770 train_time:35639ms step_avg:95.04ms
step:386/1770 train_time:35735ms step_avg:95.04ms
step:387/1770 train_time:35829ms step_avg:95.04ms
step:388/1770 train_time:35926ms step_avg:95.04ms
step:389/1770 train_time:36023ms step_avg:95.05ms
step:390/1770 train_time:36118ms step_avg:95.05ms
step:391/1770 train_time:36214ms step_avg:95.05ms
step:392/1770 train_time:36309ms step_avg:95.05ms
step:393/1770 train_time:36404ms step_avg:95.05ms
step:394/1770 train_time:36499ms step_avg:95.05ms
step:395/1770 train_time:36594ms step_avg:95.05ms
step:396/1770 train_time:36691ms step_avg:95.05ms
step:397/1770 train_time:36788ms step_avg:95.06ms
step:398/1770 train_time:36885ms step_avg:95.06ms
step:399/1770 train_time:36983ms step_avg:95.07ms
step:400/1770 train_time:37080ms step_avg:95.08ms
step:401/1770 train_time:37179ms step_avg:95.09ms
step:402/1770 train_time:37276ms step_avg:95.09ms
step:403/1770 train_time:37374ms step_avg:95.10ms
step:404/1770 train_time:37471ms step_avg:95.10ms
step:405/1770 train_time:37567ms step_avg:95.11ms
step:406/1770 train_time:37664ms step_avg:95.11ms
step:407/1770 train_time:37761ms step_avg:95.12ms
step:408/1770 train_time:37858ms step_avg:95.12ms
step:409/1770 train_time:37956ms step_avg:95.13ms
step:410/1770 train_time:38054ms step_avg:95.13ms
step:411/1770 train_time:38151ms step_avg:95.14ms
step:412/1770 train_time:38249ms step_avg:95.15ms
step:413/1770 train_time:38346ms step_avg:95.15ms
step:414/1770 train_time:38443ms step_avg:95.16ms
step:415/1770 train_time:38541ms step_avg:95.16ms
step:416/1770 train_time:38638ms step_avg:95.17ms
step:417/1770 train_time:38735ms step_avg:95.17ms
step:418/1770 train_time:38832ms step_avg:95.18ms
step:419/1770 train_time:38929ms step_avg:95.18ms
step:420/1770 train_time:39027ms step_avg:95.19ms
step:421/1770 train_time:39125ms step_avg:95.19ms
step:422/1770 train_time:39222ms step_avg:95.20ms
step:423/1770 train_time:39320ms step_avg:95.21ms
step:424/1770 train_time:39417ms step_avg:95.21ms
step:425/1770 train_time:39514ms step_avg:95.22ms
step:426/1770 train_time:39611ms step_avg:95.22ms
step:427/1770 train_time:39708ms step_avg:95.22ms
step:428/1770 train_time:39805ms step_avg:95.23ms
step:429/1770 train_time:39902ms step_avg:95.23ms
step:430/1770 train_time:40000ms step_avg:95.24ms
step:431/1770 train_time:40097ms step_avg:95.24ms
step:432/1770 train_time:40196ms step_avg:95.25ms
step:433/1770 train_time:40295ms step_avg:95.26ms
step:434/1770 train_time:40392ms step_avg:95.26ms
step:435/1770 train_time:40489ms step_avg:95.27ms
step:436/1770 train_time:40587ms step_avg:95.27ms
step:437/1770 train_time:40684ms step_avg:95.28ms
step:438/1770 train_time:40782ms step_avg:95.28ms
step:439/1770 train_time:40879ms step_avg:95.29ms
step:440/1770 train_time:40976ms step_avg:95.29ms
step:441/1770 train_time:41074ms step_avg:95.30ms
step:442/1770 train_time:41171ms step_avg:95.30ms
step:443/1770 train_time:41268ms step_avg:95.31ms
step:444/1770 train_time:41366ms step_avg:95.31ms
step:445/1770 train_time:41463ms step_avg:95.32ms
step:446/1770 train_time:41560ms step_avg:95.32ms
step:447/1770 train_time:41658ms step_avg:95.33ms
step:448/1770 train_time:41756ms step_avg:95.33ms
step:449/1770 train_time:41854ms step_avg:95.34ms
step:450/1770 train_time:41951ms step_avg:95.34ms
step:451/1770 train_time:42048ms step_avg:95.35ms
step:452/1770 train_time:42145ms step_avg:95.35ms
step:453/1770 train_time:42243ms step_avg:95.36ms
step:454/1770 train_time:42341ms step_avg:95.36ms
step:455/1770 train_time:42439ms step_avg:95.37ms
step:456/1770 train_time:42536ms step_avg:95.37ms
step:457/1770 train_time:42634ms step_avg:95.38ms
step:458/1770 train_time:42731ms step_avg:95.38ms
step:459/1770 train_time:42828ms step_avg:95.39ms
step:460/1770 train_time:42925ms step_avg:95.39ms
step:461/1770 train_time:43022ms step_avg:95.39ms
step:462/1770 train_time:43120ms step_avg:95.40ms
step:463/1770 train_time:43217ms step_avg:95.40ms
step:464/1770 train_time:43314ms step_avg:95.41ms
step:465/1770 train_time:43412ms step_avg:95.41ms
step:466/1770 train_time:43509ms step_avg:95.41ms
step:467/1770 train_time:43606ms step_avg:95.42ms
step:468/1770 train_time:43704ms step_avg:95.42ms
step:469/1770 train_time:43801ms step_avg:95.43ms
step:470/1770 train_time:43898ms step_avg:95.43ms
step:471/1770 train_time:43996ms step_avg:95.44ms
step:472/1770 train_time:44093ms step_avg:95.44ms
step:473/1770 train_time:44191ms step_avg:95.44ms
step:474/1770 train_time:44288ms step_avg:95.45ms
step:475/1770 train_time:44386ms step_avg:95.45ms
step:476/1770 train_time:44484ms step_avg:95.46ms
step:477/1770 train_time:44581ms step_avg:95.46ms
step:478/1770 train_time:44679ms step_avg:95.47ms
step:479/1770 train_time:44777ms step_avg:95.47ms
step:480/1770 train_time:44875ms step_avg:95.48ms
step:481/1770 train_time:44973ms step_avg:95.48ms
step:482/1770 train_time:45069ms step_avg:95.49ms
step:483/1770 train_time:45167ms step_avg:95.49ms
step:484/1770 train_time:45264ms step_avg:95.49ms
step:485/1770 train_time:45361ms step_avg:95.50ms
step:486/1770 train_time:45459ms step_avg:95.50ms
step:487/1770 train_time:45557ms step_avg:95.51ms
step:488/1770 train_time:45654ms step_avg:95.51ms
step:489/1770 train_time:45751ms step_avg:95.51ms
step:490/1770 train_time:45849ms step_avg:95.52ms
step:491/1770 train_time:45946ms step_avg:95.52ms
step:492/1770 train_time:46044ms step_avg:95.53ms
step:493/1770 train_time:46141ms step_avg:95.53ms
step:494/1770 train_time:46238ms step_avg:95.53ms
step:495/1770 train_time:46336ms step_avg:95.54ms
step:496/1770 train_time:46433ms step_avg:95.54ms
step:497/1770 train_time:46529ms step_avg:95.54ms
step:498/1770 train_time:46627ms step_avg:95.55ms
step:499/1770 train_time:46724ms step_avg:95.55ms
step:500/1770 train_time:46822ms step_avg:95.56ms
step:500/1770 val_loss:3.7540 train_time:46918ms step_avg:95.75ms
step:501/1770 train_time:46939ms step_avg:95.60ms
step:502/1770 train_time:47026ms step_avg:95.58ms
step:503/1770 train_time:47126ms step_avg:95.59ms
step:504/1770 train_time:47223ms step_avg:95.59ms
step:505/1770 train_time:47320ms step_avg:95.60ms
step:506/1770 train_time:47417ms step_avg:95.60ms
step:507/1770 train_time:47514ms step_avg:95.60ms
step:508/1770 train_time:47610ms step_avg:95.60ms
step:509/1770 train_time:47708ms step_avg:95.61ms
step:510/1770 train_time:47804ms step_avg:95.61ms
step:511/1770 train_time:47901ms step_avg:95.61ms
step:512/1770 train_time:47999ms step_avg:95.62ms
step:513/1770 train_time:48097ms step_avg:95.62ms
step:514/1770 train_time:48196ms step_avg:95.63ms
step:515/1770 train_time:48293ms step_avg:95.63ms
step:516/1770 train_time:48391ms step_avg:95.63ms
step:517/1770 train_time:48489ms step_avg:95.64ms
step:518/1770 train_time:48587ms step_avg:95.64ms
step:519/1770 train_time:48684ms step_avg:95.65ms
step:520/1770 train_time:48780ms step_avg:95.65ms
step:521/1770 train_time:48877ms step_avg:95.65ms
step:522/1770 train_time:48974ms step_avg:95.65ms
step:523/1770 train_time:49072ms step_avg:95.66ms
step:524/1770 train_time:49170ms step_avg:95.66ms
step:525/1770 train_time:49268ms step_avg:95.67ms
step:526/1770 train_time:49366ms step_avg:95.67ms
step:527/1770 train_time:49463ms step_avg:95.67ms
step:528/1770 train_time:49561ms step_avg:95.68ms
step:529/1770 train_time:49658ms step_avg:95.68ms
step:530/1770 train_time:49755ms step_avg:95.68ms
step:531/1770 train_time:49853ms step_avg:95.69ms
step:532/1770 train_time:49951ms step_avg:95.69ms
step:533/1770 train_time:50048ms step_avg:95.69ms
step:534/1770 train_time:50146ms step_avg:95.70ms
step:535/1770 train_time:50244ms step_avg:95.70ms
step:536/1770 train_time:50342ms step_avg:95.71ms
step:537/1770 train_time:50440ms step_avg:95.71ms
step:538/1770 train_time:50538ms step_avg:95.72ms
step:539/1770 train_time:50636ms step_avg:95.72ms
step:540/1770 train_time:50733ms step_avg:95.72ms
step:541/1770 train_time:50831ms step_avg:95.73ms
step:542/1770 train_time:50928ms step_avg:95.73ms
step:543/1770 train_time:51026ms step_avg:95.73ms
step:544/1770 train_time:51124ms step_avg:95.74ms
step:545/1770 train_time:51222ms step_avg:95.74ms
step:546/1770 train_time:51319ms step_avg:95.74ms
step:547/1770 train_time:51417ms step_avg:95.75ms
step:548/1770 train_time:51515ms step_avg:95.75ms
step:549/1770 train_time:51613ms step_avg:95.76ms
step:550/1770 train_time:51711ms step_avg:95.76ms
step:551/1770 train_time:51808ms step_avg:95.76ms
step:552/1770 train_time:51906ms step_avg:95.77ms
step:553/1770 train_time:52006ms step_avg:95.78ms
step:554/1770 train_time:52104ms step_avg:95.78ms
step:555/1770 train_time:52200ms step_avg:95.78ms
step:556/1770 train_time:52298ms step_avg:95.78ms
step:557/1770 train_time:52395ms step_avg:95.79ms
step:558/1770 train_time:52494ms step_avg:95.79ms
step:559/1770 train_time:52592ms step_avg:95.80ms
step:560/1770 train_time:52691ms step_avg:95.80ms
step:561/1770 train_time:52789ms step_avg:95.81ms
step:562/1770 train_time:52887ms step_avg:95.81ms
step:563/1770 train_time:52985ms step_avg:95.81ms
step:564/1770 train_time:53083ms step_avg:95.82ms
step:565/1770 train_time:53180ms step_avg:95.82ms
step:566/1770 train_time:53277ms step_avg:95.82ms
step:567/1770 train_time:53374ms step_avg:95.82ms
step:568/1770 train_time:53472ms step_avg:95.83ms
step:569/1770 train_time:53569ms step_avg:95.83ms
step:570/1770 train_time:53666ms step_avg:95.83ms
step:571/1770 train_time:53764ms step_avg:95.84ms
step:572/1770 train_time:53862ms step_avg:95.84ms
step:573/1770 train_time:53961ms step_avg:95.84ms
step:574/1770 train_time:54059ms step_avg:95.85ms
step:575/1770 train_time:54157ms step_avg:95.85ms
step:576/1770 train_time:54255ms step_avg:95.86ms
step:577/1770 train_time:54353ms step_avg:95.86ms
step:578/1770 train_time:54451ms step_avg:95.86ms
step:579/1770 train_time:54548ms step_avg:95.87ms
step:580/1770 train_time:54646ms step_avg:95.87ms
step:581/1770 train_time:54743ms step_avg:95.87ms
step:582/1770 train_time:54841ms step_avg:95.88ms
step:583/1770 train_time:54939ms step_avg:95.88ms
step:584/1770 train_time:55037ms step_avg:95.88ms
step:585/1770 train_time:55134ms step_avg:95.89ms
step:586/1770 train_time:55233ms step_avg:95.89ms
step:587/1770 train_time:55331ms step_avg:95.89ms
step:588/1770 train_time:55429ms step_avg:95.90ms
step:589/1770 train_time:55526ms step_avg:95.90ms
step:590/1770 train_time:55623ms step_avg:95.90ms
step:591/1770 train_time:55722ms step_avg:95.91ms
step:592/1770 train_time:55819ms step_avg:95.91ms
step:593/1770 train_time:55917ms step_avg:95.91ms
step:594/1770 train_time:56015ms step_avg:95.92ms
step:595/1770 train_time:56112ms step_avg:95.92ms
step:596/1770 train_time:56210ms step_avg:95.92ms
step:597/1770 train_time:56309ms step_avg:95.93ms
step:598/1770 train_time:56406ms step_avg:95.93ms
step:599/1770 train_time:56504ms step_avg:95.93ms
step:600/1770 train_time:56601ms step_avg:95.93ms
step:601/1770 train_time:56698ms step_avg:95.94ms
step:602/1770 train_time:56796ms step_avg:95.94ms
step:603/1770 train_time:56894ms step_avg:95.94ms
step:604/1770 train_time:56992ms step_avg:95.95ms
step:605/1770 train_time:57091ms step_avg:95.95ms
step:606/1770 train_time:57189ms step_avg:95.96ms
step:607/1770 train_time:57287ms step_avg:95.96ms
step:608/1770 train_time:57385ms step_avg:95.96ms
step:609/1770 train_time:57483ms step_avg:95.96ms
step:610/1770 train_time:57580ms step_avg:95.97ms
step:611/1770 train_time:57679ms step_avg:95.97ms
step:612/1770 train_time:57776ms step_avg:95.97ms
step:613/1770 train_time:57874ms step_avg:95.98ms
step:614/1770 train_time:57972ms step_avg:95.98ms
step:615/1770 train_time:58069ms step_avg:95.98ms
step:616/1770 train_time:58167ms step_avg:95.99ms
step:617/1770 train_time:58265ms step_avg:95.99ms
step:618/1770 train_time:58363ms step_avg:95.99ms
step:619/1770 train_time:58461ms step_avg:95.99ms
step:620/1770 train_time:58558ms step_avg:96.00ms
step:621/1770 train_time:58656ms step_avg:96.00ms
step:622/1770 train_time:58754ms step_avg:96.00ms
step:623/1770 train_time:58853ms step_avg:96.01ms
step:624/1770 train_time:58952ms step_avg:96.01ms
step:625/1770 train_time:59049ms step_avg:96.01ms
step:625/1770 val_loss:3.6688 train_time:59145ms step_avg:96.17ms
step:626/1770 train_time:59166ms step_avg:96.05ms
step:627/1770 train_time:59255ms step_avg:96.04ms
step:628/1770 train_time:59359ms step_avg:96.05ms
step:629/1770 train_time:59458ms step_avg:96.05ms
step:630/1770 train_time:59555ms step_avg:96.06ms
step:631/1770 train_time:59652ms step_avg:96.06ms
step:632/1770 train_time:59749ms step_avg:96.06ms
step:633/1770 train_time:59846ms step_avg:96.06ms
step:634/1770 train_time:59943ms step_avg:96.06ms
step:635/1770 train_time:60041ms step_avg:96.07ms
step:636/1770 train_time:60139ms step_avg:96.07ms
step:637/1770 train_time:60238ms step_avg:96.07ms
step:638/1770 train_time:60337ms step_avg:96.08ms
step:639/1770 train_time:60436ms step_avg:96.08ms
step:640/1770 train_time:60534ms step_avg:96.09ms
step:641/1770 train_time:60631ms step_avg:96.09ms
step:642/1770 train_time:60728ms step_avg:96.09ms
step:643/1770 train_time:60826ms step_avg:96.09ms
step:644/1770 train_time:60924ms step_avg:96.09ms
step:645/1770 train_time:61021ms step_avg:96.10ms
step:646/1770 train_time:61118ms step_avg:96.10ms
step:647/1770 train_time:61216ms step_avg:96.10ms
step:648/1770 train_time:61313ms step_avg:96.10ms
step:649/1770 train_time:61411ms step_avg:96.10ms
step:650/1770 train_time:61509ms step_avg:96.11ms
step:651/1770 train_time:61606ms step_avg:96.11ms
step:652/1770 train_time:61704ms step_avg:96.11ms
step:653/1770 train_time:61801ms step_avg:96.11ms
step:654/1770 train_time:61899ms step_avg:96.12ms
step:655/1770 train_time:61996ms step_avg:96.12ms
step:656/1770 train_time:62093ms step_avg:96.12ms
step:657/1770 train_time:62190ms step_avg:96.12ms
step:658/1770 train_time:62289ms step_avg:96.13ms
step:659/1770 train_time:62389ms step_avg:96.13ms
step:660/1770 train_time:62488ms step_avg:96.13ms
step:661/1770 train_time:62587ms step_avg:96.14ms
step:662/1770 train_time:62686ms step_avg:96.14ms
step:663/1770 train_time:62786ms step_avg:96.15ms
step:664/1770 train_time:62886ms step_avg:96.16ms
step:665/1770 train_time:62985ms step_avg:96.16ms
step:666/1770 train_time:63085ms step_avg:96.17ms
step:667/1770 train_time:63185ms step_avg:96.17ms
step:668/1770 train_time:63285ms step_avg:96.18ms
step:669/1770 train_time:63385ms step_avg:96.18ms
step:670/1770 train_time:63485ms step_avg:96.19ms
step:671/1770 train_time:63584ms step_avg:96.19ms
step:672/1770 train_time:63684ms step_avg:96.20ms
step:673/1770 train_time:63783ms step_avg:96.20ms
step:674/1770 train_time:63883ms step_avg:96.21ms
step:675/1770 train_time:63983ms step_avg:96.22ms
step:676/1770 train_time:64083ms step_avg:96.22ms
step:677/1770 train_time:64183ms step_avg:96.23ms
step:678/1770 train_time:64284ms step_avg:96.23ms
step:679/1770 train_time:64385ms step_avg:96.24ms
step:680/1770 train_time:64485ms step_avg:96.25ms
step:681/1770 train_time:64585ms step_avg:96.25ms
step:682/1770 train_time:64684ms step_avg:96.26ms
step:683/1770 train_time:64785ms step_avg:96.26ms
step:684/1770 train_time:64885ms step_avg:96.27ms
step:685/1770 train_time:64985ms step_avg:96.27ms
step:686/1770 train_time:65085ms step_avg:96.28ms
step:687/1770 train_time:65185ms step_avg:96.29ms
step:688/1770 train_time:65285ms step_avg:96.29ms
step:689/1770 train_time:65386ms step_avg:96.30ms
step:690/1770 train_time:65486ms step_avg:96.30ms
step:691/1770 train_time:65586ms step_avg:96.31ms
step:692/1770 train_time:65686ms step_avg:96.31ms
step:693/1770 train_time:65785ms step_avg:96.32ms
step:694/1770 train_time:65885ms step_avg:96.32ms
step:695/1770 train_time:65985ms step_avg:96.33ms
step:696/1770 train_time:66085ms step_avg:96.33ms
step:697/1770 train_time:66185ms step_avg:96.34ms
step:698/1770 train_time:66284ms step_avg:96.34ms
step:699/1770 train_time:66384ms step_avg:96.35ms
step:700/1770 train_time:66485ms step_avg:96.35ms
step:701/1770 train_time:66585ms step_avg:96.36ms
step:702/1770 train_time:66685ms step_avg:96.37ms
step:703/1770 train_time:66785ms step_avg:96.37ms
step:704/1770 train_time:66885ms step_avg:96.38ms
step:705/1770 train_time:66986ms step_avg:96.38ms
step:706/1770 train_time:67086ms step_avg:96.39ms
step:707/1770 train_time:67186ms step_avg:96.39ms
step:708/1770 train_time:67286ms step_avg:96.40ms
step:709/1770 train_time:67386ms step_avg:96.40ms
step:710/1770 train_time:67486ms step_avg:96.41ms
step:711/1770 train_time:67585ms step_avg:96.41ms
step:712/1770 train_time:67685ms step_avg:96.42ms
step:713/1770 train_time:67785ms step_avg:96.42ms
step:714/1770 train_time:67885ms step_avg:96.43ms
step:715/1770 train_time:67986ms step_avg:96.43ms
step:716/1770 train_time:68085ms step_avg:96.44ms
step:717/1770 train_time:68185ms step_avg:96.44ms
step:718/1770 train_time:68285ms step_avg:96.45ms
step:719/1770 train_time:68385ms step_avg:96.45ms
step:720/1770 train_time:68485ms step_avg:96.46ms
step:721/1770 train_time:68585ms step_avg:96.46ms
step:722/1770 train_time:68685ms step_avg:96.47ms
step:723/1770 train_time:68784ms step_avg:96.47ms
step:724/1770 train_time:68884ms step_avg:96.48ms
step:725/1770 train_time:68984ms step_avg:96.48ms
step:726/1770 train_time:69084ms step_avg:96.49ms
step:727/1770 train_time:69184ms step_avg:96.49ms
step:728/1770 train_time:69284ms step_avg:96.50ms
step:729/1770 train_time:69384ms step_avg:96.50ms
step:730/1770 train_time:69483ms step_avg:96.50ms
step:731/1770 train_time:69583ms step_avg:96.51ms
step:732/1770 train_time:69683ms step_avg:96.51ms
step:733/1770 train_time:69782ms step_avg:96.52ms
step:734/1770 train_time:69881ms step_avg:96.52ms
step:735/1770 train_time:69981ms step_avg:96.53ms
step:736/1770 train_time:70080ms step_avg:96.53ms
step:737/1770 train_time:70179ms step_avg:96.53ms
step:738/1770 train_time:70279ms step_avg:96.54ms
step:739/1770 train_time:70378ms step_avg:96.54ms
step:740/1770 train_time:70477ms step_avg:96.54ms
step:741/1770 train_time:70576ms step_avg:96.55ms
step:742/1770 train_time:70675ms step_avg:96.55ms
step:743/1770 train_time:70775ms step_avg:96.55ms
step:744/1770 train_time:70874ms step_avg:96.56ms
step:745/1770 train_time:70973ms step_avg:96.56ms
step:746/1770 train_time:71073ms step_avg:96.57ms
step:747/1770 train_time:71172ms step_avg:96.57ms
step:748/1770 train_time:71270ms step_avg:96.57ms
step:749/1770 train_time:71369ms step_avg:96.58ms
step:750/1770 train_time:71468ms step_avg:96.58ms
step:750/1770 val_loss:3.6031 train_time:71565ms step_avg:96.71ms
step:751/1770 train_time:71587ms step_avg:96.61ms
step:752/1770 train_time:71679ms step_avg:96.60ms
step:753/1770 train_time:71780ms step_avg:96.61ms
step:754/1770 train_time:71880ms step_avg:96.61ms
step:755/1770 train_time:71979ms step_avg:96.62ms
step:756/1770 train_time:72078ms step_avg:96.62ms
step:757/1770 train_time:72177ms step_avg:96.62ms
step:758/1770 train_time:72276ms step_avg:96.63ms
step:759/1770 train_time:72374ms step_avg:96.63ms
step:760/1770 train_time:72473ms step_avg:96.63ms
step:761/1770 train_time:72574ms step_avg:96.64ms
step:762/1770 train_time:72675ms step_avg:96.64ms
step:763/1770 train_time:72777ms step_avg:96.65ms
step:764/1770 train_time:72876ms step_avg:96.65ms
step:765/1770 train_time:72977ms step_avg:96.66ms
step:766/1770 train_time:73076ms step_avg:96.66ms
step:767/1770 train_time:73175ms step_avg:96.66ms
step:768/1770 train_time:73275ms step_avg:96.67ms
step:769/1770 train_time:73374ms step_avg:96.67ms
step:770/1770 train_time:73473ms step_avg:96.67ms
step:771/1770 train_time:73572ms step_avg:96.68ms
step:772/1770 train_time:73672ms step_avg:96.68ms
step:773/1770 train_time:73773ms step_avg:96.69ms
step:774/1770 train_time:73872ms step_avg:96.69ms
step:775/1770 train_time:73972ms step_avg:96.70ms
step:776/1770 train_time:74072ms step_avg:96.70ms
step:777/1770 train_time:74172ms step_avg:96.70ms
step:778/1770 train_time:74270ms step_avg:96.71ms
step:779/1770 train_time:74369ms step_avg:96.71ms
step:780/1770 train_time:74467ms step_avg:96.71ms
step:781/1770 train_time:74566ms step_avg:96.71ms
step:782/1770 train_time:74665ms step_avg:96.72ms
step:783/1770 train_time:74764ms step_avg:96.72ms
step:784/1770 train_time:74864ms step_avg:96.72ms
step:785/1770 train_time:74963ms step_avg:96.73ms
step:786/1770 train_time:75062ms step_avg:96.73ms
step:787/1770 train_time:75161ms step_avg:96.73ms
step:788/1770 train_time:75260ms step_avg:96.74ms
step:789/1770 train_time:75360ms step_avg:96.74ms
step:790/1770 train_time:75460ms step_avg:96.74ms
step:791/1770 train_time:75560ms step_avg:96.75ms
step:792/1770 train_time:75660ms step_avg:96.75ms
step:793/1770 train_time:75760ms step_avg:96.76ms
step:794/1770 train_time:75860ms step_avg:96.76ms
step:795/1770 train_time:75960ms step_avg:96.76ms
step:796/1770 train_time:76061ms step_avg:96.77ms
step:797/1770 train_time:76161ms step_avg:96.77ms
step:798/1770 train_time:76260ms step_avg:96.78ms
step:799/1770 train_time:76360ms step_avg:96.78ms
step:800/1770 train_time:76459ms step_avg:96.78ms
step:801/1770 train_time:76559ms step_avg:96.79ms
step:802/1770 train_time:76659ms step_avg:96.79ms
step:803/1770 train_time:76759ms step_avg:96.80ms
step:804/1770 train_time:76860ms step_avg:96.80ms
step:805/1770 train_time:76960ms step_avg:96.81ms
step:806/1770 train_time:77060ms step_avg:96.81ms
step:807/1770 train_time:77160ms step_avg:96.81ms
step:808/1770 train_time:77261ms step_avg:96.82ms
step:809/1770 train_time:77360ms step_avg:96.82ms
step:810/1770 train_time:77460ms step_avg:96.83ms
step:811/1770 train_time:77561ms step_avg:96.83ms
step:812/1770 train_time:77660ms step_avg:96.83ms
step:813/1770 train_time:77759ms step_avg:96.84ms
step:814/1770 train_time:77859ms step_avg:96.84ms
step:815/1770 train_time:77959ms step_avg:96.84ms
step:816/1770 train_time:78060ms step_avg:96.85ms
step:817/1770 train_time:78159ms step_avg:96.85ms
step:818/1770 train_time:78260ms step_avg:96.86ms
step:819/1770 train_time:78360ms step_avg:96.86ms
step:820/1770 train_time:78461ms step_avg:96.87ms
step:821/1770 train_time:78560ms step_avg:96.87ms
step:822/1770 train_time:78660ms step_avg:96.87ms
step:823/1770 train_time:78760ms step_avg:96.88ms
step:824/1770 train_time:78860ms step_avg:96.88ms
step:825/1770 train_time:78960ms step_avg:96.88ms
step:826/1770 train_time:79060ms step_avg:96.89ms
step:827/1770 train_time:79160ms step_avg:96.89ms
step:828/1770 train_time:79260ms step_avg:96.89ms
step:829/1770 train_time:79360ms step_avg:96.90ms
step:830/1770 train_time:79460ms step_avg:96.90ms
step:831/1770 train_time:79559ms step_avg:96.91ms
step:832/1770 train_time:79660ms step_avg:96.91ms
step:833/1770 train_time:79761ms step_avg:96.91ms
step:834/1770 train_time:79861ms step_avg:96.92ms
step:835/1770 train_time:79961ms step_avg:96.92ms
step:836/1770 train_time:80060ms step_avg:96.93ms
step:837/1770 train_time:80160ms step_avg:96.93ms
step:838/1770 train_time:80260ms step_avg:96.93ms
step:839/1770 train_time:80360ms step_avg:96.94ms
step:840/1770 train_time:80460ms step_avg:96.94ms
step:841/1770 train_time:80559ms step_avg:96.94ms
step:842/1770 train_time:80659ms step_avg:96.95ms
step:843/1770 train_time:80759ms step_avg:96.95ms
step:844/1770 train_time:80858ms step_avg:96.95ms
step:845/1770 train_time:80959ms step_avg:96.96ms
step:846/1770 train_time:81059ms step_avg:96.96ms
step:847/1770 train_time:81159ms step_avg:96.96ms
step:848/1770 train_time:81259ms step_avg:96.97ms
step:849/1770 train_time:81359ms step_avg:96.97ms
step:850/1770 train_time:81459ms step_avg:96.97ms
step:851/1770 train_time:81558ms step_avg:96.98ms
step:852/1770 train_time:81658ms step_avg:96.98ms
step:853/1770 train_time:81758ms step_avg:96.98ms
step:854/1770 train_time:81858ms step_avg:96.99ms
step:855/1770 train_time:81958ms step_avg:96.99ms
step:856/1770 train_time:82058ms step_avg:97.00ms
step:857/1770 train_time:82158ms step_avg:97.00ms
step:858/1770 train_time:82259ms step_avg:97.00ms
step:859/1770 train_time:82360ms step_avg:97.01ms
step:860/1770 train_time:82460ms step_avg:97.01ms
step:861/1770 train_time:82560ms step_avg:97.02ms
step:862/1770 train_time:82660ms step_avg:97.02ms
step:863/1770 train_time:82761ms step_avg:97.02ms
step:864/1770 train_time:82860ms step_avg:97.03ms
step:865/1770 train_time:82960ms step_avg:97.03ms
step:866/1770 train_time:83060ms step_avg:97.03ms
step:867/1770 train_time:83160ms step_avg:97.04ms
step:868/1770 train_time:83260ms step_avg:97.04ms
step:869/1770 train_time:83360ms step_avg:97.04ms
step:870/1770 train_time:83460ms step_avg:97.05ms
step:871/1770 train_time:83559ms step_avg:97.05ms
step:872/1770 train_time:83660ms step_avg:97.05ms
step:873/1770 train_time:83760ms step_avg:97.06ms
step:874/1770 train_time:83860ms step_avg:97.06ms
step:875/1770 train_time:83959ms step_avg:97.06ms
step:875/1770 val_loss:3.5542 train_time:84057ms step_avg:97.18ms
step:876/1770 train_time:84079ms step_avg:97.09ms
step:877/1770 train_time:84169ms step_avg:97.08ms
step:878/1770 train_time:84270ms step_avg:97.09ms
step:879/1770 train_time:84370ms step_avg:97.09ms
step:880/1770 train_time:84469ms step_avg:97.09ms
step:881/1770 train_time:84569ms step_avg:97.09ms
step:882/1770 train_time:84668ms step_avg:97.10ms
step:883/1770 train_time:84767ms step_avg:97.10ms
step:884/1770 train_time:84866ms step_avg:97.10ms
step:885/1770 train_time:84964ms step_avg:97.10ms
step:886/1770 train_time:85065ms step_avg:97.11ms
step:887/1770 train_time:85165ms step_avg:97.11ms
step:888/1770 train_time:85267ms step_avg:97.11ms
step:889/1770 train_time:85367ms step_avg:97.12ms
step:890/1770 train_time:85468ms step_avg:97.12ms
step:891/1770 train_time:85568ms step_avg:97.13ms
step:892/1770 train_time:85667ms step_avg:97.13ms
step:893/1770 train_time:85766ms step_avg:97.13ms
step:894/1770 train_time:85866ms step_avg:97.13ms
step:895/1770 train_time:85966ms step_avg:97.14ms
step:896/1770 train_time:86064ms step_avg:97.14ms
step:897/1770 train_time:86164ms step_avg:97.14ms
step:898/1770 train_time:86265ms step_avg:97.14ms
step:899/1770 train_time:86365ms step_avg:97.15ms
step:900/1770 train_time:86464ms step_avg:97.15ms
step:901/1770 train_time:86563ms step_avg:97.15ms
step:902/1770 train_time:86663ms step_avg:97.16ms
step:903/1770 train_time:86763ms step_avg:97.16ms
step:904/1770 train_time:86862ms step_avg:97.16ms
step:905/1770 train_time:86960ms step_avg:97.16ms
step:906/1770 train_time:87060ms step_avg:97.17ms
step:907/1770 train_time:87159ms step_avg:97.17ms
step:908/1770 train_time:87259ms step_avg:97.17ms
step:909/1770 train_time:87359ms step_avg:97.17ms
step:910/1770 train_time:87458ms step_avg:97.18ms
step:911/1770 train_time:87558ms step_avg:97.18ms
step:912/1770 train_time:87657ms step_avg:97.18ms
step:913/1770 train_time:87756ms step_avg:97.18ms
step:914/1770 train_time:87856ms step_avg:97.19ms
step:915/1770 train_time:87955ms step_avg:97.19ms
step:916/1770 train_time:88054ms step_avg:97.19ms
step:917/1770 train_time:88154ms step_avg:97.19ms
step:918/1770 train_time:88254ms step_avg:97.20ms
step:919/1770 train_time:88353ms step_avg:97.20ms
step:920/1770 train_time:88455ms step_avg:97.20ms
step:921/1770 train_time:88557ms step_avg:97.21ms
step:922/1770 train_time:88658ms step_avg:97.21ms
step:923/1770 train_time:88758ms step_avg:97.22ms
step:924/1770 train_time:88859ms step_avg:97.22ms
step:925/1770 train_time:88959ms step_avg:97.22ms
step:926/1770 train_time:89059ms step_avg:97.23ms
step:927/1770 train_time:89159ms step_avg:97.23ms
step:928/1770 train_time:89260ms step_avg:97.23ms
step:929/1770 train_time:89360ms step_avg:97.24ms
step:930/1770 train_time:89460ms step_avg:97.24ms
step:931/1770 train_time:89559ms step_avg:97.24ms
step:932/1770 train_time:89660ms step_avg:97.24ms
step:933/1770 train_time:89761ms step_avg:97.25ms
step:934/1770 train_time:89862ms step_avg:97.25ms
step:935/1770 train_time:89962ms step_avg:97.26ms
step:936/1770 train_time:90062ms step_avg:97.26ms
step:937/1770 train_time:90163ms step_avg:97.26ms
step:938/1770 train_time:90264ms step_avg:97.27ms
step:939/1770 train_time:90367ms step_avg:97.27ms
step:940/1770 train_time:90469ms step_avg:97.28ms
step:941/1770 train_time:90571ms step_avg:97.28ms
step:942/1770 train_time:90673ms step_avg:97.29ms
step:943/1770 train_time:90775ms step_avg:97.29ms
step:944/1770 train_time:90876ms step_avg:97.30ms
step:945/1770 train_time:90978ms step_avg:97.30ms
step:946/1770 train_time:91079ms step_avg:97.31ms
step:947/1770 train_time:91180ms step_avg:97.31ms
step:948/1770 train_time:91280ms step_avg:97.31ms
step:949/1770 train_time:91380ms step_avg:97.32ms
step:950/1770 train_time:91481ms step_avg:97.32ms
step:951/1770 train_time:91581ms step_avg:97.32ms
step:952/1770 train_time:91682ms step_avg:97.33ms
step:953/1770 train_time:91783ms step_avg:97.33ms
step:954/1770 train_time:91884ms step_avg:97.33ms
step:955/1770 train_time:91985ms step_avg:97.34ms
step:956/1770 train_time:92085ms step_avg:97.34ms
step:957/1770 train_time:92186ms step_avg:97.35ms
step:958/1770 train_time:92287ms step_avg:97.35ms
step:959/1770 train_time:92388ms step_avg:97.35ms
step:960/1770 train_time:92489ms step_avg:97.36ms
step:961/1770 train_time:92591ms step_avg:97.36ms
step:962/1770 train_time:92695ms step_avg:97.37ms
step:963/1770 train_time:92796ms step_avg:97.37ms
step:964/1770 train_time:92897ms step_avg:97.38ms
step:965/1770 train_time:92998ms step_avg:97.38ms
step:966/1770 train_time:93099ms step_avg:97.38ms
step:967/1770 train_time:93199ms step_avg:97.39ms
step:968/1770 train_time:93300ms step_avg:97.39ms
step:969/1770 train_time:93400ms step_avg:97.39ms
step:970/1770 train_time:93501ms step_avg:97.40ms
step:971/1770 train_time:93602ms step_avg:97.40ms
step:972/1770 train_time:93703ms step_avg:97.40ms
step:973/1770 train_time:93805ms step_avg:97.41ms
step:974/1770 train_time:93906ms step_avg:97.41ms
step:975/1770 train_time:94009ms step_avg:97.42ms
step:976/1770 train_time:94111ms step_avg:97.42ms
step:977/1770 train_time:94212ms step_avg:97.43ms
step:978/1770 train_time:94312ms step_avg:97.43ms
step:979/1770 train_time:94414ms step_avg:97.43ms
step:980/1770 train_time:94516ms step_avg:97.44ms
step:981/1770 train_time:94617ms step_avg:97.44ms
step:982/1770 train_time:94718ms step_avg:97.45ms
step:983/1770 train_time:94818ms step_avg:97.45ms
step:984/1770 train_time:94920ms step_avg:97.45ms
step:985/1770 train_time:95021ms step_avg:97.46ms
step:986/1770 train_time:95122ms step_avg:97.46ms
step:987/1770 train_time:95223ms step_avg:97.46ms
step:988/1770 train_time:95324ms step_avg:97.47ms
step:989/1770 train_time:95425ms step_avg:97.47ms
step:990/1770 train_time:95527ms step_avg:97.48ms
step:991/1770 train_time:95629ms step_avg:97.48ms
step:992/1770 train_time:95730ms step_avg:97.48ms
step:993/1770 train_time:95831ms step_avg:97.49ms
step:994/1770 train_time:95932ms step_avg:97.49ms
step:995/1770 train_time:96034ms step_avg:97.50ms
step:996/1770 train_time:96136ms step_avg:97.50ms
step:997/1770 train_time:96237ms step_avg:97.50ms
step:998/1770 train_time:96338ms step_avg:97.51ms
step:999/1770 train_time:96438ms step_avg:97.51ms
step:1000/1770 train_time:96539ms step_avg:97.51ms
step:1000/1770 val_loss:3.5147 train_time:96637ms step_avg:97.61ms
step:1001/1770 train_time:96659ms step_avg:97.54ms
step:1002/1770 train_time:96751ms step_avg:97.53ms
step:1003/1770 train_time:96854ms step_avg:97.54ms
step:1004/1770 train_time:96954ms step_avg:97.54ms
step:1005/1770 train_time:97055ms step_avg:97.54ms
step:1006/1770 train_time:97154ms step_avg:97.54ms
step:1007/1770 train_time:97255ms step_avg:97.55ms
step:1008/1770 train_time:97355ms step_avg:97.55ms
step:1009/1770 train_time:97455ms step_avg:97.55ms
step:1010/1770 train_time:97556ms step_avg:97.56ms
step:1011/1770 train_time:97658ms step_avg:97.56ms
step:1012/1770 train_time:97760ms step_avg:97.56ms
step:1013/1770 train_time:97861ms step_avg:97.57ms
step:1014/1770 train_time:97961ms step_avg:97.57ms
step:1015/1770 train_time:98061ms step_avg:97.57ms
step:1016/1770 train_time:98162ms step_avg:97.58ms
step:1017/1770 train_time:98263ms step_avg:97.58ms
step:1018/1770 train_time:98365ms step_avg:97.58ms
step:1019/1770 train_time:98466ms step_avg:97.59ms
step:1020/1770 train_time:98568ms step_avg:97.59ms
step:1021/1770 train_time:98669ms step_avg:97.60ms
step:1022/1770 train_time:98770ms step_avg:97.60ms
step:1023/1770 train_time:98872ms step_avg:97.60ms
step:1024/1770 train_time:98974ms step_avg:97.61ms
step:1025/1770 train_time:99075ms step_avg:97.61ms
step:1026/1770 train_time:99176ms step_avg:97.61ms
step:1027/1770 train_time:99279ms step_avg:97.62ms
step:1028/1770 train_time:99380ms step_avg:97.62ms
step:1029/1770 train_time:99481ms step_avg:97.63ms
step:1030/1770 train_time:99581ms step_avg:97.63ms
step:1031/1770 train_time:99681ms step_avg:97.63ms
step:1032/1770 train_time:99781ms step_avg:97.63ms
step:1033/1770 train_time:99881ms step_avg:97.64ms
step:1034/1770 train_time:99983ms step_avg:97.64ms
step:1035/1770 train_time:100085ms step_avg:97.64ms
step:1036/1770 train_time:100187ms step_avg:97.65ms
step:1037/1770 train_time:100287ms step_avg:97.65ms
step:1038/1770 train_time:100389ms step_avg:97.65ms
step:1039/1770 train_time:100490ms step_avg:97.66ms
step:1040/1770 train_time:100591ms step_avg:97.66ms
step:1041/1770 train_time:100693ms step_avg:97.67ms
step:1042/1770 train_time:100795ms step_avg:97.67ms
step:1043/1770 train_time:100897ms step_avg:97.67ms
step:1044/1770 train_time:100999ms step_avg:97.68ms
step:1045/1770 train_time:101099ms step_avg:97.68ms
step:1046/1770 train_time:101199ms step_avg:97.68ms
step:1047/1770 train_time:101300ms step_avg:97.69ms
step:1048/1770 train_time:101400ms step_avg:97.69ms
step:1049/1770 train_time:101503ms step_avg:97.69ms
step:1050/1770 train_time:101604ms step_avg:97.70ms
step:1051/1770 train_time:101706ms step_avg:97.70ms
step:1052/1770 train_time:101807ms step_avg:97.70ms
step:1053/1770 train_time:101907ms step_avg:97.71ms
step:1054/1770 train_time:102008ms step_avg:97.71ms
step:1055/1770 train_time:102111ms step_avg:97.71ms
step:1056/1770 train_time:102213ms step_avg:97.72ms
step:1057/1770 train_time:102315ms step_avg:97.72ms
step:1058/1770 train_time:102417ms step_avg:97.73ms
step:1059/1770 train_time:102518ms step_avg:97.73ms
step:1060/1770 train_time:102619ms step_avg:97.73ms
step:1061/1770 train_time:102719ms step_avg:97.73ms
step:1062/1770 train_time:102820ms step_avg:97.74ms
step:1063/1770 train_time:102923ms step_avg:97.74ms
step:1064/1770 train_time:103024ms step_avg:97.75ms
step:1065/1770 train_time:103124ms step_avg:97.75ms
step:1066/1770 train_time:103227ms step_avg:97.75ms
step:1067/1770 train_time:103329ms step_avg:97.76ms
step:1068/1770 train_time:103431ms step_avg:97.76ms
step:1069/1770 train_time:103532ms step_avg:97.76ms
step:1070/1770 train_time:103634ms step_avg:97.77ms
step:1071/1770 train_time:103736ms step_avg:97.77ms
step:1072/1770 train_time:103837ms step_avg:97.78ms
step:1073/1770 train_time:103938ms step_avg:97.78ms
step:1074/1770 train_time:104039ms step_avg:97.78ms
step:1075/1770 train_time:104140ms step_avg:97.78ms
step:1076/1770 train_time:104241ms step_avg:97.79ms
step:1077/1770 train_time:104342ms step_avg:97.79ms
step:1078/1770 train_time:104442ms step_avg:97.79ms
step:1079/1770 train_time:104542ms step_avg:97.79ms
step:1080/1770 train_time:104643ms step_avg:97.80ms
step:1081/1770 train_time:104744ms step_avg:97.80ms
step:1082/1770 train_time:104846ms step_avg:97.80ms
step:1083/1770 train_time:104947ms step_avg:97.81ms
step:1084/1770 train_time:105051ms step_avg:97.81ms
step:1085/1770 train_time:105152ms step_avg:97.82ms
step:1086/1770 train_time:105253ms step_avg:97.82ms
step:1087/1770 train_time:105354ms step_avg:97.82ms
step:1088/1770 train_time:105456ms step_avg:97.83ms
step:1089/1770 train_time:105558ms step_avg:97.83ms
step:1090/1770 train_time:105659ms step_avg:97.83ms
step:1091/1770 train_time:105759ms step_avg:97.83ms
step:1092/1770 train_time:105860ms step_avg:97.84ms
step:1093/1770 train_time:105960ms step_avg:97.84ms
step:1094/1770 train_time:106061ms step_avg:97.84ms
step:1095/1770 train_time:106162ms step_avg:97.85ms
step:1096/1770 train_time:106263ms step_avg:97.85ms
step:1097/1770 train_time:106366ms step_avg:97.85ms
step:1098/1770 train_time:106468ms step_avg:97.86ms
step:1099/1770 train_time:106570ms step_avg:97.86ms
step:1100/1770 train_time:106671ms step_avg:97.86ms
step:1101/1770 train_time:106773ms step_avg:97.87ms
step:1102/1770 train_time:106874ms step_avg:97.87ms
step:1103/1770 train_time:106976ms step_avg:97.87ms
step:1104/1770 train_time:107078ms step_avg:97.88ms
step:1105/1770 train_time:107179ms step_avg:97.88ms
step:1106/1770 train_time:107280ms step_avg:97.88ms
step:1107/1770 train_time:107381ms step_avg:97.89ms
step:1108/1770 train_time:107481ms step_avg:97.89ms
step:1109/1770 train_time:107582ms step_avg:97.89ms
step:1110/1770 train_time:107683ms step_avg:97.89ms
step:1111/1770 train_time:107785ms step_avg:97.90ms
step:1112/1770 train_time:107889ms step_avg:97.90ms
step:1113/1770 train_time:107990ms step_avg:97.91ms
step:1114/1770 train_time:108092ms step_avg:97.91ms
step:1115/1770 train_time:108193ms step_avg:97.91ms
step:1116/1770 train_time:108296ms step_avg:97.92ms
step:1117/1770 train_time:108398ms step_avg:97.92ms
step:1118/1770 train_time:108498ms step_avg:97.92ms
step:1119/1770 train_time:108599ms step_avg:97.92ms
step:1120/1770 train_time:108698ms step_avg:97.93ms
step:1121/1770 train_time:108799ms step_avg:97.93ms
step:1122/1770 train_time:108900ms step_avg:97.93ms
step:1123/1770 train_time:109000ms step_avg:97.93ms
step:1124/1770 train_time:109100ms step_avg:97.94ms
step:1125/1770 train_time:109201ms step_avg:97.94ms
step:1125/1770 val_loss:3.4749 train_time:109300ms step_avg:98.03ms
step:1126/1770 train_time:109322ms step_avg:97.96ms
step:1127/1770 train_time:109416ms step_avg:97.96ms
step:1128/1770 train_time:109518ms step_avg:97.96ms
step:1129/1770 train_time:109619ms step_avg:97.96ms
step:1130/1770 train_time:109719ms step_avg:97.96ms
step:1131/1770 train_time:109820ms step_avg:97.97ms
step:1132/1770 train_time:109921ms step_avg:97.97ms
step:1133/1770 train_time:110021ms step_avg:97.97ms
step:1134/1770 train_time:110121ms step_avg:97.97ms
step:1135/1770 train_time:110223ms step_avg:97.98ms
step:1136/1770 train_time:110327ms step_avg:97.98ms
step:1137/1770 train_time:110430ms step_avg:97.99ms
step:1138/1770 train_time:110531ms step_avg:97.99ms
step:1139/1770 train_time:110633ms step_avg:97.99ms
step:1140/1770 train_time:110735ms step_avg:98.00ms
step:1141/1770 train_time:110835ms step_avg:98.00ms
step:1142/1770 train_time:110936ms step_avg:98.00ms
step:1143/1770 train_time:111036ms step_avg:98.00ms
step:1144/1770 train_time:111137ms step_avg:98.00ms
step:1145/1770 train_time:111238ms step_avg:98.01ms
step:1146/1770 train_time:111338ms step_avg:98.01ms
step:1147/1770 train_time:111440ms step_avg:98.01ms
step:1148/1770 train_time:111543ms step_avg:98.02ms
step:1149/1770 train_time:111645ms step_avg:98.02ms
step:1150/1770 train_time:111747ms step_avg:98.02ms
step:1151/1770 train_time:111849ms step_avg:98.03ms
step:1152/1770 train_time:111952ms step_avg:98.03ms
step:1153/1770 train_time:112053ms step_avg:98.03ms
step:1154/1770 train_time:112154ms step_avg:98.04ms
step:1155/1770 train_time:112255ms step_avg:98.04ms
step:1156/1770 train_time:112355ms step_avg:98.04ms
step:1157/1770 train_time:112458ms step_avg:98.05ms
step:1158/1770 train_time:112560ms step_avg:98.05ms
step:1159/1770 train_time:112660ms step_avg:98.05ms
step:1160/1770 train_time:112762ms step_avg:98.05ms
step:1161/1770 train_time:112863ms step_avg:98.06ms
step:1162/1770 train_time:112964ms step_avg:98.06ms
step:1163/1770 train_time:113065ms step_avg:98.06ms
step:1164/1770 train_time:113166ms step_avg:98.06ms
step:1165/1770 train_time:113268ms step_avg:98.07ms
step:1166/1770 train_time:113369ms step_avg:98.07ms
step:1167/1770 train_time:113471ms step_avg:98.07ms
step:1168/1770 train_time:113575ms step_avg:98.08ms
step:1169/1770 train_time:113675ms step_avg:98.08ms
step:1170/1770 train_time:113776ms step_avg:98.08ms
step:1171/1770 train_time:113877ms step_avg:98.09ms
step:1172/1770 train_time:113977ms step_avg:98.09ms
step:1173/1770 train_time:114078ms step_avg:98.09ms
step:1174/1770 train_time:114179ms step_avg:98.09ms
step:1175/1770 train_time:114280ms step_avg:98.09ms
step:1176/1770 train_time:114382ms step_avg:98.10ms
step:1177/1770 train_time:114484ms step_avg:98.10ms
step:1178/1770 train_time:114586ms step_avg:98.10ms
step:1179/1770 train_time:114688ms step_avg:98.11ms
step:1180/1770 train_time:114789ms step_avg:98.11ms
step:1181/1770 train_time:114891ms step_avg:98.11ms
step:1182/1770 train_time:114992ms step_avg:98.12ms
step:1183/1770 train_time:115095ms step_avg:98.12ms
step:1184/1770 train_time:115197ms step_avg:98.12ms
step:1185/1770 train_time:115299ms step_avg:98.13ms
step:1186/1770 train_time:115402ms step_avg:98.13ms
step:1187/1770 train_time:115506ms step_avg:98.14ms
step:1188/1770 train_time:115607ms step_avg:98.14ms
step:1189/1770 train_time:115709ms step_avg:98.14ms
step:1190/1770 train_time:115812ms step_avg:98.15ms
step:1191/1770 train_time:115915ms step_avg:98.15ms
step:1192/1770 train_time:116017ms step_avg:98.15ms
step:1193/1770 train_time:116119ms step_avg:98.16ms
step:1194/1770 train_time:116220ms step_avg:98.16ms
step:1195/1770 train_time:116322ms step_avg:98.16ms
step:1196/1770 train_time:116425ms step_avg:98.17ms
step:1197/1770 train_time:116527ms step_avg:98.17ms
step:1198/1770 train_time:116629ms step_avg:98.17ms
step:1199/1770 train_time:116732ms step_avg:98.18ms
step:1200/1770 train_time:116836ms step_avg:98.18ms
step:1201/1770 train_time:116938ms step_avg:98.18ms
step:1202/1770 train_time:117039ms step_avg:98.19ms
step:1203/1770 train_time:117141ms step_avg:98.19ms
step:1204/1770 train_time:117244ms step_avg:98.19ms
step:1205/1770 train_time:117346ms step_avg:98.20ms
step:1206/1770 train_time:117450ms step_avg:98.20ms
step:1207/1770 train_time:117552ms step_avg:98.21ms
step:1208/1770 train_time:117654ms step_avg:98.21ms
step:1209/1770 train_time:117756ms step_avg:98.21ms
step:1210/1770 train_time:117858ms step_avg:98.21ms
step:1211/1770 train_time:117961ms step_avg:98.22ms
step:1212/1770 train_time:118065ms step_avg:98.22ms
step:1213/1770 train_time:118167ms step_avg:98.23ms
step:1214/1770 train_time:118270ms step_avg:98.23ms
step:1215/1770 train_time:118373ms step_avg:98.23ms
step:1216/1770 train_time:118477ms step_avg:98.24ms
step:1217/1770 train_time:118579ms step_avg:98.24ms
step:1218/1770 train_time:118681ms step_avg:98.25ms
step:1219/1770 train_time:118782ms step_avg:98.25ms
step:1220/1770 train_time:118885ms step_avg:98.25ms
step:1221/1770 train_time:118987ms step_avg:98.26ms
step:1222/1770 train_time:119091ms step_avg:98.26ms
step:1223/1770 train_time:119193ms step_avg:98.26ms
step:1224/1770 train_time:119295ms step_avg:98.27ms
step:1225/1770 train_time:119398ms step_avg:98.27ms
step:1226/1770 train_time:119499ms step_avg:98.27ms
step:1227/1770 train_time:119603ms step_avg:98.28ms
step:1228/1770 train_time:119706ms step_avg:98.28ms
step:1229/1770 train_time:119809ms step_avg:98.28ms
step:1230/1770 train_time:119911ms step_avg:98.29ms
step:1231/1770 train_time:120013ms step_avg:98.29ms
step:1232/1770 train_time:120115ms step_avg:98.29ms
step:1233/1770 train_time:120217ms step_avg:98.30ms
step:1234/1770 train_time:120320ms step_avg:98.30ms
step:1235/1770 train_time:120422ms step_avg:98.30ms
step:1236/1770 train_time:120527ms step_avg:98.31ms
step:1237/1770 train_time:120629ms step_avg:98.31ms
step:1238/1770 train_time:120731ms step_avg:98.32ms
step:1239/1770 train_time:120833ms step_avg:98.32ms
step:1240/1770 train_time:120936ms step_avg:98.32ms
step:1241/1770 train_time:121038ms step_avg:98.33ms
step:1242/1770 train_time:121140ms step_avg:98.33ms
step:1243/1770 train_time:121242ms step_avg:98.33ms
step:1244/1770 train_time:121344ms step_avg:98.33ms
step:1245/1770 train_time:121446ms step_avg:98.34ms
step:1246/1770 train_time:121550ms step_avg:98.34ms
step:1247/1770 train_time:121652ms step_avg:98.34ms
step:1248/1770 train_time:121755ms step_avg:98.35ms
step:1249/1770 train_time:121856ms step_avg:98.35ms
step:1250/1770 train_time:121958ms step_avg:98.35ms
step:1250/1770 val_loss:3.4266 train_time:122060ms step_avg:98.44ms
step:1251/1770 train_time:122082ms step_avg:98.37ms
step:1252/1770 train_time:122176ms step_avg:98.37ms
step:1253/1770 train_time:122282ms step_avg:98.38ms
step:1254/1770 train_time:122384ms step_avg:98.38ms
step:1255/1770 train_time:122488ms step_avg:98.38ms
step:1256/1770 train_time:122589ms step_avg:98.39ms
step:1257/1770 train_time:122691ms step_avg:98.39ms
step:1258/1770 train_time:122792ms step_avg:98.39ms
step:1259/1770 train_time:122895ms step_avg:98.39ms
step:1260/1770 train_time:122996ms step_avg:98.40ms
step:1261/1770 train_time:123101ms step_avg:98.40ms
step:1262/1770 train_time:123204ms step_avg:98.41ms
step:1263/1770 train_time:123307ms step_avg:98.41ms
step:1264/1770 train_time:123410ms step_avg:98.41ms
step:1265/1770 train_time:123512ms step_avg:98.42ms
step:1266/1770 train_time:123613ms step_avg:98.42ms
step:1267/1770 train_time:123716ms step_avg:98.42ms
step:1268/1770 train_time:123819ms step_avg:98.43ms
step:1269/1770 train_time:123920ms step_avg:98.43ms
step:1270/1770 train_time:124025ms step_avg:98.43ms
step:1271/1770 train_time:124127ms step_avg:98.44ms
step:1272/1770 train_time:124229ms step_avg:98.44ms
step:1273/1770 train_time:124331ms step_avg:98.44ms
step:1274/1770 train_time:124433ms step_avg:98.44ms
step:1275/1770 train_time:124536ms step_avg:98.45ms
step:1276/1770 train_time:124638ms step_avg:98.45ms
step:1277/1770 train_time:124740ms step_avg:98.45ms
step:1278/1770 train_time:124843ms step_avg:98.46ms
step:1279/1770 train_time:124946ms step_avg:98.46ms
step:1280/1770 train_time:125049ms step_avg:98.46ms
step:1281/1770 train_time:125151ms step_avg:98.47ms
step:1282/1770 train_time:125253ms step_avg:98.47ms
step:1283/1770 train_time:125356ms step_avg:98.47ms
step:1284/1770 train_time:125459ms step_avg:98.48ms
step:1285/1770 train_time:125560ms step_avg:98.48ms
step:1286/1770 train_time:125663ms step_avg:98.48ms
step:1287/1770 train_time:125767ms step_avg:98.49ms
step:1288/1770 train_time:125869ms step_avg:98.49ms
step:1289/1770 train_time:125971ms step_avg:98.49ms
step:1290/1770 train_time:126072ms step_avg:98.49ms
step:1291/1770 train_time:126174ms step_avg:98.50ms
step:1292/1770 train_time:126276ms step_avg:98.50ms
step:1293/1770 train_time:126379ms step_avg:98.50ms
step:1294/1770 train_time:126481ms step_avg:98.51ms
step:1295/1770 train_time:126584ms step_avg:98.51ms
step:1296/1770 train_time:126687ms step_avg:98.51ms
step:1297/1770 train_time:126789ms step_avg:98.52ms
step:1298/1770 train_time:126891ms step_avg:98.52ms
step:1299/1770 train_time:126994ms step_avg:98.52ms
step:1300/1770 train_time:127095ms step_avg:98.52ms
step:1301/1770 train_time:127198ms step_avg:98.53ms
step:1302/1770 train_time:127301ms step_avg:98.53ms
step:1303/1770 train_time:127403ms step_avg:98.53ms
step:1304/1770 train_time:127505ms step_avg:98.54ms
step:1305/1770 train_time:127608ms step_avg:98.54ms
step:1306/1770 train_time:127710ms step_avg:98.54ms
step:1307/1770 train_time:127812ms step_avg:98.54ms
step:1308/1770 train_time:127914ms step_avg:98.55ms
step:1309/1770 train_time:128017ms step_avg:98.55ms
step:1310/1770 train_time:128120ms step_avg:98.55ms
step:1311/1770 train_time:128222ms step_avg:98.56ms
step:1312/1770 train_time:128324ms step_avg:98.56ms
step:1313/1770 train_time:128426ms step_avg:98.56ms
step:1314/1770 train_time:128528ms step_avg:98.56ms
step:1315/1770 train_time:128630ms step_avg:98.57ms
step:1316/1770 train_time:128732ms step_avg:98.57ms
step:1317/1770 train_time:128835ms step_avg:98.57ms
step:1318/1770 train_time:128939ms step_avg:98.58ms
step:1319/1770 train_time:129043ms step_avg:98.58ms
step:1320/1770 train_time:129146ms step_avg:98.58ms
step:1321/1770 train_time:129248ms step_avg:98.59ms
step:1322/1770 train_time:129350ms step_avg:98.59ms
step:1323/1770 train_time:129452ms step_avg:98.59ms
step:1324/1770 train_time:129555ms step_avg:98.60ms
step:1325/1770 train_time:129658ms step_avg:98.60ms
step:1326/1770 train_time:129760ms step_avg:98.60ms
step:1327/1770 train_time:129865ms step_avg:98.61ms
step:1328/1770 train_time:129967ms step_avg:98.61ms
step:1329/1770 train_time:130069ms step_avg:98.61ms
step:1330/1770 train_time:130171ms step_avg:98.61ms
step:1331/1770 train_time:130272ms step_avg:98.62ms
step:1332/1770 train_time:130373ms step_avg:98.62ms
step:1333/1770 train_time:130475ms step_avg:98.62ms
step:1334/1770 train_time:130577ms step_avg:98.62ms
step:1335/1770 train_time:130679ms step_avg:98.63ms
step:1336/1770 train_time:130782ms step_avg:98.63ms
step:1337/1770 train_time:130885ms step_avg:98.63ms
step:1338/1770 train_time:130987ms step_avg:98.64ms
step:1339/1770 train_time:131090ms step_avg:98.64ms
step:1340/1770 train_time:131193ms step_avg:98.64ms
step:1341/1770 train_time:131296ms step_avg:98.64ms
step:1342/1770 train_time:131399ms step_avg:98.65ms
step:1343/1770 train_time:131504ms step_avg:98.65ms
step:1344/1770 train_time:131606ms step_avg:98.66ms
step:1345/1770 train_time:131708ms step_avg:98.66ms
step:1346/1770 train_time:131811ms step_avg:98.66ms
step:1347/1770 train_time:131913ms step_avg:98.66ms
step:1348/1770 train_time:132017ms step_avg:98.67ms
step:1349/1770 train_time:132120ms step_avg:98.67ms
step:1350/1770 train_time:132224ms step_avg:98.67ms
step:1351/1770 train_time:132328ms step_avg:98.68ms
step:1352/1770 train_time:132429ms step_avg:98.68ms
step:1353/1770 train_time:132533ms step_avg:98.68ms
step:1354/1770 train_time:132635ms step_avg:98.69ms
step:1355/1770 train_time:132737ms step_avg:98.69ms
step:1356/1770 train_time:132839ms step_avg:98.69ms
step:1357/1770 train_time:132941ms step_avg:98.69ms
step:1358/1770 train_time:133044ms step_avg:98.70ms
step:1359/1770 train_time:133146ms step_avg:98.70ms
step:1360/1770 train_time:133250ms step_avg:98.70ms
step:1361/1770 train_time:133353ms step_avg:98.71ms
step:1362/1770 train_time:133456ms step_avg:98.71ms
step:1363/1770 train_time:133559ms step_avg:98.71ms
step:1364/1770 train_time:133663ms step_avg:98.72ms
step:1365/1770 train_time:133766ms step_avg:98.72ms
step:1366/1770 train_time:133867ms step_avg:98.72ms
step:1367/1770 train_time:133969ms step_avg:98.72ms
step:1368/1770 train_time:134070ms step_avg:98.73ms
step:1369/1770 train_time:134173ms step_avg:98.73ms
step:1370/1770 train_time:134276ms step_avg:98.73ms
step:1371/1770 train_time:134378ms step_avg:98.73ms
step:1372/1770 train_time:134481ms step_avg:98.74ms
step:1373/1770 train_time:134584ms step_avg:98.74ms
step:1374/1770 train_time:134688ms step_avg:98.75ms
step:1375/1770 train_time:134791ms step_avg:98.75ms
step:1375/1770 val_loss:3.3835 train_time:134891ms step_avg:98.82ms
step:1376/1770 train_time:134913ms step_avg:98.77ms
step:1377/1770 train_time:135010ms step_avg:98.76ms
step:1378/1770 train_time:135113ms step_avg:98.77ms
step:1379/1770 train_time:135215ms step_avg:98.77ms
step:1380/1770 train_time:135317ms step_avg:98.77ms
step:1381/1770 train_time:135419ms step_avg:98.77ms
step:1382/1770 train_time:135520ms step_avg:98.78ms
step:1383/1770 train_time:135622ms step_avg:98.78ms
step:1384/1770 train_time:135725ms step_avg:98.78ms
step:1385/1770 train_time:135827ms step_avg:98.78ms
step:1386/1770 train_time:135931ms step_avg:98.79ms
step:1387/1770 train_time:136035ms step_avg:98.79ms
step:1388/1770 train_time:136137ms step_avg:98.79ms
step:1389/1770 train_time:136240ms step_avg:98.80ms
step:1390/1770 train_time:136343ms step_avg:98.80ms
step:1391/1770 train_time:136445ms step_avg:98.80ms
step:1392/1770 train_time:136547ms step_avg:98.80ms
step:1393/1770 train_time:136649ms step_avg:98.81ms
step:1394/1770 train_time:136752ms step_avg:98.81ms
step:1395/1770 train_time:136856ms step_avg:98.81ms
step:1396/1770 train_time:136959ms step_avg:98.82ms
step:1397/1770 train_time:137062ms step_avg:98.82ms
step:1398/1770 train_time:137164ms step_avg:98.82ms
step:1399/1770 train_time:137267ms step_avg:98.82ms
step:1400/1770 train_time:137369ms step_avg:98.83ms
step:1401/1770 train_time:137472ms step_avg:98.83ms
step:1402/1770 train_time:137575ms step_avg:98.83ms
step:1403/1770 train_time:137676ms step_avg:98.83ms
step:1404/1770 train_time:137779ms step_avg:98.84ms
step:1405/1770 train_time:137882ms step_avg:98.84ms
step:1406/1770 train_time:137984ms step_avg:98.84ms
step:1407/1770 train_time:138086ms step_avg:98.84ms
step:1408/1770 train_time:138190ms step_avg:98.85ms
step:1409/1770 train_time:138294ms step_avg:98.85ms
step:1410/1770 train_time:138396ms step_avg:98.85ms
step:1411/1770 train_time:138498ms step_avg:98.86ms
step:1412/1770 train_time:138600ms step_avg:98.86ms
step:1413/1770 train_time:138701ms step_avg:98.86ms
step:1414/1770 train_time:138805ms step_avg:98.86ms
step:1415/1770 train_time:138908ms step_avg:98.87ms
step:1416/1770 train_time:139012ms step_avg:98.87ms
step:1417/1770 train_time:139116ms step_avg:98.87ms
step:1418/1770 train_time:139217ms step_avg:98.88ms
step:1419/1770 train_time:139320ms step_avg:98.88ms
step:1420/1770 train_time:139423ms step_avg:98.88ms
step:1421/1770 train_time:139525ms step_avg:98.88ms
step:1422/1770 train_time:139627ms step_avg:98.89ms
step:1423/1770 train_time:139731ms step_avg:98.89ms
step:1424/1770 train_time:139834ms step_avg:98.89ms
step:1425/1770 train_time:139936ms step_avg:98.89ms
step:1426/1770 train_time:140039ms step_avg:98.90ms
step:1427/1770 train_time:140141ms step_avg:98.90ms
step:1428/1770 train_time:140246ms step_avg:98.90ms
step:1429/1770 train_time:140349ms step_avg:98.91ms
step:1430/1770 train_time:140452ms step_avg:98.91ms
step:1431/1770 train_time:140556ms step_avg:98.91ms
step:1432/1770 train_time:140658ms step_avg:98.92ms
step:1433/1770 train_time:140759ms step_avg:98.92ms
step:1434/1770 train_time:140861ms step_avg:98.92ms
step:1435/1770 train_time:140964ms step_avg:98.92ms
step:1436/1770 train_time:141070ms step_avg:98.93ms
step:1437/1770 train_time:141173ms step_avg:98.93ms
step:1438/1770 train_time:141276ms step_avg:98.93ms
step:1439/1770 train_time:141377ms step_avg:98.93ms
step:1440/1770 train_time:141479ms step_avg:98.94ms
step:1441/1770 train_time:141582ms step_avg:98.94ms
step:1442/1770 train_time:141686ms step_avg:98.94ms
step:1443/1770 train_time:141788ms step_avg:98.94ms
step:1444/1770 train_time:141892ms step_avg:98.95ms
step:1445/1770 train_time:141996ms step_avg:98.95ms
step:1446/1770 train_time:142099ms step_avg:98.95ms
step:1447/1770 train_time:142203ms step_avg:98.96ms
step:1448/1770 train_time:142306ms step_avg:98.96ms
step:1449/1770 train_time:142413ms step_avg:98.97ms
step:1450/1770 train_time:142516ms step_avg:98.97ms
step:1451/1770 train_time:142619ms step_avg:98.97ms
step:1452/1770 train_time:142722ms step_avg:98.97ms
step:1453/1770 train_time:142825ms step_avg:98.98ms
step:1454/1770 train_time:142929ms step_avg:98.98ms
step:1455/1770 train_time:143034ms step_avg:98.99ms
step:1456/1770 train_time:143137ms step_avg:98.99ms
step:1457/1770 train_time:143242ms step_avg:98.99ms
step:1458/1770 train_time:143346ms step_avg:99.00ms
step:1459/1770 train_time:143451ms step_avg:99.00ms
step:1460/1770 train_time:143554ms step_avg:99.00ms
step:1461/1770 train_time:143657ms step_avg:99.01ms
step:1462/1770 train_time:143760ms step_avg:99.01ms
step:1463/1770 train_time:143863ms step_avg:99.01ms
step:1464/1770 train_time:143968ms step_avg:99.02ms
step:1465/1770 train_time:144073ms step_avg:99.02ms
step:1466/1770 train_time:144176ms step_avg:99.02ms
step:1467/1770 train_time:144280ms step_avg:99.03ms
step:1468/1770 train_time:144384ms step_avg:99.03ms
step:1469/1770 train_time:144489ms step_avg:99.03ms
step:1470/1770 train_time:144592ms step_avg:99.04ms
step:1471/1770 train_time:144697ms step_avg:99.04ms
step:1472/1770 train_time:144800ms step_avg:99.04ms
step:1473/1770 train_time:144905ms step_avg:99.05ms
step:1474/1770 train_time:145010ms step_avg:99.05ms
step:1475/1770 train_time:145113ms step_avg:99.05ms
step:1476/1770 train_time:145216ms step_avg:99.06ms
step:1477/1770 train_time:145321ms step_avg:99.06ms
step:1478/1770 train_time:145425ms step_avg:99.06ms
step:1479/1770 train_time:145530ms step_avg:99.07ms
step:1480/1770 train_time:145634ms step_avg:99.07ms
step:1481/1770 train_time:145741ms step_avg:99.08ms
step:1482/1770 train_time:145844ms step_avg:99.08ms
step:1483/1770 train_time:145948ms step_avg:99.08ms
step:1484/1770 train_time:146053ms step_avg:99.09ms
step:1485/1770 train_time:146156ms step_avg:99.09ms
step:1486/1770 train_time:146259ms step_avg:99.09ms
step:1487/1770 train_time:146361ms step_avg:99.09ms
step:1488/1770 train_time:146464ms step_avg:99.10ms
step:1489/1770 train_time:146569ms step_avg:99.10ms
step:1490/1770 train_time:146673ms step_avg:99.10ms
step:1491/1770 train_time:146776ms step_avg:99.11ms
step:1492/1770 train_time:146880ms step_avg:99.11ms
step:1493/1770 train_time:146987ms step_avg:99.11ms
step:1494/1770 train_time:147094ms step_avg:99.12ms
step:1495/1770 train_time:147197ms step_avg:99.12ms
step:1496/1770 train_time:147300ms step_avg:99.13ms
step:1497/1770 train_time:147404ms step_avg:99.13ms
step:1498/1770 train_time:147506ms step_avg:99.13ms
step:1499/1770 train_time:147609ms step_avg:99.13ms
step:1500/1770 train_time:147712ms step_avg:99.14ms
step:1500/1770 val_loss:3.3452 train_time:147814ms step_avg:99.20ms
step:1501/1770 train_time:147836ms step_avg:99.15ms
step:1502/1770 train_time:147932ms step_avg:99.15ms
step:1503/1770 train_time:148035ms step_avg:99.15ms
step:1504/1770 train_time:148139ms step_avg:99.16ms
step:1505/1770 train_time:148244ms step_avg:99.16ms
step:1506/1770 train_time:148348ms step_avg:99.16ms
step:1507/1770 train_time:148452ms step_avg:99.17ms
step:1508/1770 train_time:148556ms step_avg:99.17ms
step:1509/1770 train_time:148659ms step_avg:99.17ms
step:1510/1770 train_time:148761ms step_avg:99.17ms
step:1511/1770 train_time:148866ms step_avg:99.18ms
step:1512/1770 train_time:148971ms step_avg:99.18ms
step:1513/1770 train_time:149075ms step_avg:99.19ms
step:1514/1770 train_time:149179ms step_avg:99.19ms
step:1515/1770 train_time:149281ms step_avg:99.19ms
step:1516/1770 train_time:149386ms step_avg:99.19ms
step:1517/1770 train_time:149491ms step_avg:99.20ms
step:1518/1770 train_time:149595ms step_avg:99.20ms
step:1519/1770 train_time:149697ms step_avg:99.20ms
step:1520/1770 train_time:149801ms step_avg:99.21ms
step:1521/1770 train_time:149904ms step_avg:99.21ms
step:1522/1770 train_time:150008ms step_avg:99.21ms
step:1523/1770 train_time:150113ms step_avg:99.22ms
step:1524/1770 train_time:150216ms step_avg:99.22ms
step:1525/1770 train_time:150319ms step_avg:99.22ms
step:1526/1770 train_time:150421ms step_avg:99.22ms
step:1527/1770 train_time:150525ms step_avg:99.23ms
step:1528/1770 train_time:150632ms step_avg:99.23ms
step:1529/1770 train_time:150736ms step_avg:99.23ms
step:1530/1770 train_time:150840ms step_avg:99.24ms
step:1531/1770 train_time:150943ms step_avg:99.24ms
step:1532/1770 train_time:151048ms step_avg:99.24ms
step:1533/1770 train_time:151152ms step_avg:99.25ms
step:1534/1770 train_time:151257ms step_avg:99.25ms
step:1535/1770 train_time:151360ms step_avg:99.25ms
step:1536/1770 train_time:151463ms step_avg:99.25ms
step:1537/1770 train_time:151567ms step_avg:99.26ms
step:1538/1770 train_time:151672ms step_avg:99.26ms
step:1539/1770 train_time:151775ms step_avg:99.26ms
step:1540/1770 train_time:151882ms step_avg:99.27ms
step:1541/1770 train_time:151986ms step_avg:99.27ms
step:1542/1770 train_time:152090ms step_avg:99.28ms
step:1543/1770 train_time:152194ms step_avg:99.28ms
step:1544/1770 train_time:152299ms step_avg:99.28ms
step:1545/1770 train_time:152402ms step_avg:99.28ms
step:1546/1770 train_time:152506ms step_avg:99.29ms
step:1547/1770 train_time:152609ms step_avg:99.29ms
step:1548/1770 train_time:152713ms step_avg:99.29ms
step:1549/1770 train_time:152817ms step_avg:99.30ms
step:1550/1770 train_time:152920ms step_avg:99.30ms
step:1551/1770 train_time:153023ms step_avg:99.30ms
step:1552/1770 train_time:153128ms step_avg:99.30ms
step:1553/1770 train_time:153231ms step_avg:99.31ms
step:1554/1770 train_time:153334ms step_avg:99.31ms
step:1555/1770 train_time:153439ms step_avg:99.31ms
step:1556/1770 train_time:153541ms step_avg:99.32ms
step:1557/1770 train_time:153645ms step_avg:99.32ms
step:1558/1770 train_time:153749ms step_avg:99.32ms
step:1559/1770 train_time:153853ms step_avg:99.32ms
step:1560/1770 train_time:153957ms step_avg:99.33ms
step:1561/1770 train_time:154063ms step_avg:99.33ms
step:1562/1770 train_time:154165ms step_avg:99.33ms
step:1563/1770 train_time:154269ms step_avg:99.34ms
step:1564/1770 train_time:154373ms step_avg:99.34ms
step:1565/1770 train_time:154476ms step_avg:99.34ms
step:1566/1770 train_time:154578ms step_avg:99.34ms
step:1567/1770 train_time:154682ms step_avg:99.35ms
step:1568/1770 train_time:154785ms step_avg:99.35ms
step:1569/1770 train_time:154891ms step_avg:99.35ms
step:1570/1770 train_time:154995ms step_avg:99.36ms
step:1571/1770 train_time:155098ms step_avg:99.36ms
step:1572/1770 train_time:155202ms step_avg:99.36ms
step:1573/1770 train_time:155308ms step_avg:99.37ms
step:1574/1770 train_time:155411ms step_avg:99.37ms
step:1575/1770 train_time:155514ms step_avg:99.37ms
step:1576/1770 train_time:155617ms step_avg:99.37ms
step:1577/1770 train_time:155722ms step_avg:99.38ms
step:1578/1770 train_time:155828ms step_avg:99.38ms
step:1579/1770 train_time:155932ms step_avg:99.38ms
step:1580/1770 train_time:156035ms step_avg:99.39ms
step:1581/1770 train_time:156140ms step_avg:99.39ms
step:1582/1770 train_time:156245ms step_avg:99.39ms
step:1583/1770 train_time:156349ms step_avg:99.40ms
step:1584/1770 train_time:156454ms step_avg:99.40ms
step:1585/1770 train_time:156557ms step_avg:99.40ms
step:1586/1770 train_time:156664ms step_avg:99.41ms
step:1587/1770 train_time:156768ms step_avg:99.41ms
step:1588/1770 train_time:156871ms step_avg:99.41ms
step:1589/1770 train_time:156977ms step_avg:99.42ms
step:1590/1770 train_time:157079ms step_avg:99.42ms
step:1591/1770 train_time:157182ms step_avg:99.42ms
step:1592/1770 train_time:157288ms step_avg:99.42ms
step:1593/1770 train_time:157392ms step_avg:99.43ms
step:1594/1770 train_time:157496ms step_avg:99.43ms
step:1595/1770 train_time:157599ms step_avg:99.43ms
step:1596/1770 train_time:157704ms step_avg:99.44ms
step:1597/1770 train_time:157807ms step_avg:99.44ms
step:1598/1770 train_time:157911ms step_avg:99.44ms
step:1599/1770 train_time:158016ms step_avg:99.44ms
step:1600/1770 train_time:158121ms step_avg:99.45ms
step:1601/1770 train_time:158226ms step_avg:99.45ms
step:1602/1770 train_time:158331ms step_avg:99.45ms
step:1603/1770 train_time:158435ms step_avg:99.46ms
step:1604/1770 train_time:158538ms step_avg:99.46ms
step:1605/1770 train_time:158640ms step_avg:99.46ms
step:1606/1770 train_time:158744ms step_avg:99.46ms
step:1607/1770 train_time:158852ms step_avg:99.47ms
step:1608/1770 train_time:158956ms step_avg:99.47ms
step:1609/1770 train_time:159060ms step_avg:99.47ms
step:1610/1770 train_time:159165ms step_avg:99.48ms
step:1611/1770 train_time:159271ms step_avg:99.48ms
step:1612/1770 train_time:159375ms step_avg:99.49ms
step:1613/1770 train_time:159479ms step_avg:99.49ms
step:1614/1770 train_time:159582ms step_avg:99.49ms
step:1615/1770 train_time:159686ms step_avg:99.49ms
step:1616/1770 train_time:159790ms step_avg:99.50ms
step:1617/1770 train_time:159895ms step_avg:99.50ms
step:1618/1770 train_time:160000ms step_avg:99.50ms
step:1619/1770 train_time:160104ms step_avg:99.51ms
step:1620/1770 train_time:160208ms step_avg:99.51ms
step:1621/1770 train_time:160312ms step_avg:99.51ms
step:1622/1770 train_time:160417ms step_avg:99.51ms
step:1623/1770 train_time:160522ms step_avg:99.52ms
step:1624/1770 train_time:160626ms step_avg:99.52ms
step:1625/1770 train_time:160729ms step_avg:99.52ms
step:1625/1770 val_loss:3.3110 train_time:160832ms step_avg:99.59ms
step:1626/1770 train_time:160853ms step_avg:99.54ms
step:1627/1770 train_time:160946ms step_avg:99.53ms
step:1628/1770 train_time:161050ms step_avg:99.54ms
step:1629/1770 train_time:161152ms step_avg:99.54ms
step:1630/1770 train_time:161255ms step_avg:99.54ms
step:1631/1770 train_time:161358ms step_avg:99.54ms
step:1632/1770 train_time:161461ms step_avg:99.54ms
step:1633/1770 train_time:161565ms step_avg:99.55ms
step:1634/1770 train_time:161669ms step_avg:99.55ms
step:1635/1770 train_time:161772ms step_avg:99.55ms
step:1636/1770 train_time:161876ms step_avg:99.55ms
step:1637/1770 train_time:161982ms step_avg:99.56ms
step:1638/1770 train_time:162086ms step_avg:99.56ms
step:1639/1770 train_time:162190ms step_avg:99.56ms
step:1640/1770 train_time:162294ms step_avg:99.57ms
step:1641/1770 train_time:162398ms step_avg:99.57ms
step:1642/1770 train_time:162501ms step_avg:99.57ms
step:1643/1770 train_time:162604ms step_avg:99.57ms
step:1644/1770 train_time:162710ms step_avg:99.58ms
step:1645/1770 train_time:162813ms step_avg:99.58ms
step:1646/1770 train_time:162918ms step_avg:99.58ms
step:1647/1770 train_time:163022ms step_avg:99.59ms
step:1648/1770 train_time:163126ms step_avg:99.59ms
step:1649/1770 train_time:163231ms step_avg:99.59ms
step:1650/1770 train_time:163334ms step_avg:99.59ms
step:1651/1770 train_time:163438ms step_avg:99.60ms
step:1652/1770 train_time:163541ms step_avg:99.60ms
step:1653/1770 train_time:163645ms step_avg:99.60ms
step:1654/1770 train_time:163752ms step_avg:99.61ms
step:1655/1770 train_time:163857ms step_avg:99.61ms
step:1656/1770 train_time:163961ms step_avg:99.61ms
step:1657/1770 train_time:164065ms step_avg:99.61ms
step:1658/1770 train_time:164170ms step_avg:99.62ms
step:1659/1770 train_time:164275ms step_avg:99.62ms
step:1660/1770 train_time:164377ms step_avg:99.62ms
step:1661/1770 train_time:164482ms step_avg:99.63ms
step:1662/1770 train_time:164585ms step_avg:99.63ms
step:1663/1770 train_time:164688ms step_avg:99.63ms
step:1664/1770 train_time:164792ms step_avg:99.63ms
step:1665/1770 train_time:164894ms step_avg:99.63ms
step:1666/1770 train_time:164999ms step_avg:99.64ms
step:1667/1770 train_time:165102ms step_avg:99.64ms
step:1668/1770 train_time:165205ms step_avg:99.64ms
step:1669/1770 train_time:165309ms step_avg:99.64ms
step:1670/1770 train_time:165413ms step_avg:99.65ms
step:1671/1770 train_time:165517ms step_avg:99.65ms
step:1672/1770 train_time:165621ms step_avg:99.65ms
step:1673/1770 train_time:165726ms step_avg:99.65ms
step:1674/1770 train_time:165830ms step_avg:99.66ms
step:1675/1770 train_time:165933ms step_avg:99.66ms
step:1676/1770 train_time:166038ms step_avg:99.66ms
step:1677/1770 train_time:166145ms step_avg:99.67ms
step:1678/1770 train_time:166248ms step_avg:99.67ms
step:1679/1770 train_time:166353ms step_avg:99.67ms
step:1680/1770 train_time:166456ms step_avg:99.67ms
step:1681/1770 train_time:166560ms step_avg:99.68ms
step:1682/1770 train_time:166666ms step_avg:99.68ms
step:1683/1770 train_time:166769ms step_avg:99.68ms
step:1684/1770 train_time:166873ms step_avg:99.68ms
step:1685/1770 train_time:166975ms step_avg:99.69ms
step:1686/1770 train_time:167081ms step_avg:99.69ms
step:1687/1770 train_time:167187ms step_avg:99.69ms
step:1688/1770 train_time:167291ms step_avg:99.70ms
step:1689/1770 train_time:167395ms step_avg:99.70ms
step:1690/1770 train_time:167497ms step_avg:99.70ms
step:1691/1770 train_time:167602ms step_avg:99.70ms
step:1692/1770 train_time:167707ms step_avg:99.71ms
step:1693/1770 train_time:167812ms step_avg:99.71ms
step:1694/1770 train_time:167916ms step_avg:99.71ms
step:1695/1770 train_time:168020ms step_avg:99.71ms
step:1696/1770 train_time:168124ms step_avg:99.72ms
step:1697/1770 train_time:168230ms step_avg:99.72ms
step:1698/1770 train_time:168333ms step_avg:99.72ms
step:1699/1770 train_time:168436ms step_avg:99.73ms
step:1700/1770 train_time:168539ms step_avg:99.73ms
step:1701/1770 train_time:168643ms step_avg:99.73ms
step:1702/1770 train_time:168748ms step_avg:99.73ms
step:1703/1770 train_time:168852ms step_avg:99.74ms
step:1704/1770 train_time:168957ms step_avg:99.74ms
step:1705/1770 train_time:169060ms step_avg:99.74ms
step:1706/1770 train_time:169164ms step_avg:99.74ms
step:1707/1770 train_time:169270ms step_avg:99.75ms
step:1708/1770 train_time:169374ms step_avg:99.75ms
step:1709/1770 train_time:169479ms step_avg:99.75ms
step:1710/1770 train_time:169588ms step_avg:99.76ms
step:1711/1770 train_time:169694ms step_avg:99.76ms
step:1712/1770 train_time:169798ms step_avg:99.76ms
step:1713/1770 train_time:169902ms step_avg:99.77ms
step:1714/1770 train_time:170006ms step_avg:99.77ms
step:1715/1770 train_time:170110ms step_avg:99.77ms
step:1716/1770 train_time:170215ms step_avg:99.77ms
step:1717/1770 train_time:170319ms step_avg:99.78ms
step:1718/1770 train_time:170424ms step_avg:99.78ms
step:1719/1770 train_time:170530ms step_avg:99.78ms
step:1720/1770 train_time:170636ms step_avg:99.79ms
step:1721/1770 train_time:170740ms step_avg:99.79ms
step:1722/1770 train_time:170848ms step_avg:99.79ms
step:1723/1770 train_time:170954ms step_avg:99.80ms
step:1724/1770 train_time:171060ms step_avg:99.80ms
step:1725/1770 train_time:171168ms step_avg:99.81ms
step:1726/1770 train_time:171272ms step_avg:99.81ms
step:1727/1770 train_time:171379ms step_avg:99.81ms
step:1728/1770 train_time:171484ms step_avg:99.82ms
step:1729/1770 train_time:171588ms step_avg:99.82ms
step:1730/1770 train_time:171693ms step_avg:99.82ms
step:1731/1770 train_time:171799ms step_avg:99.83ms
step:1732/1770 train_time:171902ms step_avg:99.83ms
step:1733/1770 train_time:172008ms step_avg:99.83ms
step:1734/1770 train_time:172113ms step_avg:99.83ms
step:1735/1770 train_time:172217ms step_avg:99.84ms
step:1736/1770 train_time:172321ms step_avg:99.84ms
step:1737/1770 train_time:172425ms step_avg:99.84ms
step:1738/1770 train_time:172530ms step_avg:99.84ms
step:1739/1770 train_time:172635ms step_avg:99.85ms
step:1740/1770 train_time:172739ms step_avg:99.85ms
step:1741/1770 train_time:172845ms step_avg:99.85ms
step:1742/1770 train_time:172952ms step_avg:99.86ms
step:1743/1770 train_time:173057ms step_avg:99.86ms
step:1744/1770 train_time:173161ms step_avg:99.86ms
step:1745/1770 train_time:173265ms step_avg:99.86ms
step:1746/1770 train_time:173373ms step_avg:99.87ms
step:1747/1770 train_time:173476ms step_avg:99.87ms
step:1748/1770 train_time:173582ms step_avg:99.87ms
step:1749/1770 train_time:173688ms step_avg:99.88ms
step:1750/1770 train_time:173792ms step_avg:99.88ms
step:1750/1770 val_loss:3.2838 train_time:173895ms step_avg:99.94ms
step:1751/1770 train_time:173917ms step_avg:99.89ms
step:1752/1770 train_time:174009ms step_avg:99.89ms
step:1753/1770 train_time:174116ms step_avg:99.89ms
step:1754/1770 train_time:174220ms step_avg:99.90ms
step:1755/1770 train_time:174324ms step_avg:99.90ms
step:1756/1770 train_time:174429ms step_avg:99.90ms
step:1757/1770 train_time:174533ms step_avg:99.90ms
step:1758/1770 train_time:174637ms step_avg:99.91ms
step:1759/1770 train_time:174742ms step_avg:99.91ms
step:1760/1770 train_time:174846ms step_avg:99.91ms
step:1761/1770 train_time:174953ms step_avg:99.92ms
step:1762/1770 train_time:175061ms step_avg:99.92ms
step:1763/1770 train_time:175164ms step_avg:99.92ms
step:1764/1770 train_time:175269ms step_avg:99.93ms
step:1765/1770 train_time:175374ms step_avg:99.93ms
step:1766/1770 train_time:175482ms step_avg:99.93ms
step:1767/1770 train_time:175585ms step_avg:99.93ms
step:1768/1770 train_time:175691ms step_avg:99.94ms
step:1769/1770 train_time:175794ms step_avg:99.94ms
step:1770/1770 train_time:175899ms step_avg:99.94ms
step:1770/1770 val_loss:3.2808 train_time:176003ms step_avg:100.00ms
peak memory allocated: 29898 MiB reserved: 44552 MiB