passcheck.asm

.386
.model flat, stdcall
.stack 4096

    ; My dad was in the computer industry for 45 years; in addition to
    ; his day job, he supplemented his income while my siblings and I
    ; were in college with teaching night classes at the local community
    ; college. One of the classes he taught was assembly language
    ; programming, and something he'd have his students do near the end
    ; of the term was write a program that asked the user to enter a
    ; password, read the password, then checked whether the password
    ; was correct. (extra credit if he couldn't figure out the password
    ; from your program) Anyway, here's my entry, 20-30 years late, but
    ; here it is. My dad's students were mostly using the old "INT 21h"
    ; API to interact with the user, whereas this uses modern-day
    ; Windows console APIs.
    
    ; It should be linked as a console-subsystem windows program; you can
    ; find out how to set up a MASM-based Visual Studio project at
    ; https://programminghaven.home.blog/2020/02/16/setup-an-assembly-project-on-visual-studio-2019
    ; and then just use this as the source code.
    
    ; As a hint: the password is all printable ASCII, followed by \r\n
    ; With that hint, it is possible then to decode the password

    INCLUDELIB kernel32.lib                 ; For GetStdHandle, Get/SetConsoleMode, Read/WriteFile, and ExitProcess

    ExitProcess proto, dwExitCode:dword

    GetStdHandle proto,
    nStdHandle:dword                                    ; The standard device. -10=INPUT, -11=OUTPUT, -13=ERROR

    SetConsoleMode proto,
    hConsoleHandle:dword,
    dwMode:dword

    GetConsoleMode proto,
    hConsoleHandle:dword,
    lpMode:near32

    ReadFile proto,
    hFile:dword,
    lpBuffer:near32,
    nNumberOfCharsToRead:dword,
    lpNumberOfbytesRead:near32,
    lpOverlapped:near32                                 ; leave this NULL

    WriteFile proto,
    hFile:dword,
    lpBuffer:near32,
    nNumberOfCharsToWrite:dword,
    lpNumberOfbytesWritten:near32,
    lpOverlapped:near32                                 ; leave this NULL

.data
    askprompt      db "Enter password: ", 0
    correct        db "Password correct!", 13, 10, 0
    incorrect      db "PASSWORD INCORRECT", 13, 10, 0
    crlf           db 13d, 10d, 0
    lencheckparams dd 0fbb8a750h, 07654327h
    passparamCount db 7d
    ; the second constant beginning with 04c might be worth feeding to Google
    passparams     dd 0ff207606h, 04c11db7h,
                      0b64c0e4dh, 08dh,
                      036aba5c7h, 0173h,
                      026b545fdh, 020dh,
                      0fa90eb01h, 025bh,
                      0f37548f7h, 025dh,
                      06b5a05ddh, 0475h

.code
main PROC
    LOCAL stdinHandle
    LOCAL oldConsoleMode:dword
    invoke GetStdHandle, -10d
    mov stdinHandle, eax
    lea esi, oldConsoleMode
    invoke GetConsoleMode, eax, esi

    lea esi, askprompt
    call PrintString

    invoke SetConsoleMode, stdinHandle, 3d  ; This means "line editing, but no echo"

    sub esp, 80d
    mov edi, esp
    mov eax, 70d
    call GetString
    mov byte ptr [edi+eax], 0

    invoke SetConsoleMode, stdinHandle, oldConsoleMode

    lea esi, crlf
    call PrintString
    call PrintString

    mov esi, edi

    lea edi, lencheckparams
    mov eax, [edi]
    mov ecx, [edi + 4]
    mov bl, 07Fh
    call CheckPass
    test eax, eax
    jnz sayFailed

    lea edi, passparams
    xor edx, edx
    mov dl, passparamCount
    dec edx

checkLoop:
    xor ebx, ebx
    mov eax, [edi + 8*edx]
    mov ecx, [edi + 8*edx + 4]
    call CheckPass
    
    test eax, eax
    jnz sayFailed
    dec edx
    jns checkLoop

    lea esi, correct
    call PrintString
    mov eax, 0

    jmp doneSaying

sayFailed:
    lea esi, incorrect
    call PrintString
    mov eax, 1

doneSaying:
    add esp, 80d
    INVOKE ExitProcess, eax
    ret
main ENDP

; PrintString takes null-terminated string in ESI, prints it
PrintString PROC PRIVATE
    LOCAL myStrLen:DWORD
    LOCAL written:DWORD
    push eax
    push ebx
    push esi
    xor eax, eax

sizeloop:
    cmp byte ptr [esi], 0
    jz endSizeLoop
    inc esi
    inc eax
    jmp sizeLoop
endSizeLoop:

    mov myStrLen, eax
    invoke GetStdHandle, -11d
    pop esi
    lea ebx, written
    invoke WriteFile, eax, esi, myStrLen, ebx, 0
    pop ebx
    pop eax
    ret
PrintString ENDP

; GetString gets a string from stdin into where EDI points, max EAX chars
; return length in EAX
GetString PROC PRIVATE
    LOCAL readMax:DWORD
    LOCAL readLength:DWORD
    push ebx
    mov readMax, eax
    invoke GetStdHandle, -10d
    lea ebx, readLength
    invoke ReadFile, eax, edi, readMax, ebx, 0
    mov eax, readLength
    pop ebx
    ret
GetString ENDP

; tuning parameters in EAX and ECX, input mask in BL, null-terminated password in ESI
; retval in EAX
CheckPass PROC PRIVATE
    LOCAL inpmask:BYTE
    mov inpmask, bl
    push esi
    push edx
    push ebx

cpStrLoop:
    cmp byte ptr [esi], 0
    jz endCpStrLoop
    mov bl, byte ptr [esi]
    or bl, inpmask
    mov dl, 8
    test bl, bl
    jns cpByteLoop0         ; require low bit-8 in password: ASCII only!
    mov eax, 0ffffffffh
    jmp endCpStrLoop

cpByteLoop0:
    rcl bl, 1
    rcl eax, 1
    jnc skipXor
    xor eax, ecx
skipXor:
    dec dl
    jnz cpByteLoop0

    inc esi
    jmp cpStrLoop
endCpStrLoop:

    pop ebx
    pop edx
    pop esi
    ret
CheckPass ENDP

END main                                ; specify the entry point

passcheck.s

.intel_syntax noprefix
.code32

# My dad was in the computer industry for 45 years; in addition to
# his day job, he supplemented his income while my siblings and I
# were in college with teaching night classes at the local community
# college. One of the classes he taught was assembly language
# programming, and something he'd have his students do near the end
# of the term was write a program that asked the user to enter a
# password, read the password, then checked whether the password
# was correct. (extra credit if he couldn't figure out the password
# from your program) Anyway, here's my entry, 20-30 years late, but
# here it is. My dad's students were mostly using the old "INT 21h"
# DOS API to interact with the user, whereas this is written for
# linux using the "int 0x80" syscall interface. (the only syscalls
# used are read, write, and ioctl)

# to compile, on linux use:
# gcc -o passcheck -m32 -no-pie -nostdlib -g passcheck.s

# As a hint: the password is all printable ASCII, followed by \r\n
# (this linux version of the code replaces the trailing \n with a
# trailing \r\n to match the windows code)

# I believe that with that hint, it is possible then to decode the password

.section .rodata
    askprompt:      .ascii "Enter password: \0"
    correct:        .ascii "Password correct!\r\n\0"
    incorrect:      .ascii "PASSWORD INCORRECT\r\n\0"
    crlf:           .ascii "\r\n\0"
    lencheckparams: .int 0xfbb8a750, 0x7654327
    passparamCount: .byte 7
    # the second constant beginning with 0x4c might be worth feeding to Google
    passparams:     .int 0xff207606, 0x4c11db7
                    .int 0xb64c0e4d, 0x8d
                    .int 0x36aba5c7, 0x173
                    .int 0x26b545fd, 0x20d
                    .int 0xfa90eb01, 0x25b
                    .int 0xf37548f7, 0x25d
                    .int 0x6b5a05dd, 0x475

.text
.global _start

_start:
    sub esp, 160
    mov ebp, esp

    lea esi, askprompt
    call PrintString

    lea edx, [ebp]
    mov eax, 0x36
    xor ebx, ebx
    mov ecx, 0x5401
    push eax
    int 0x80            # ioctl TCGETS

    andb [ebp+12], (0xFF & (~8))  # Not echo

    inc ecx
    pop eax
    push eax
    int 0x80            # ioctl TCSETS

    lea edi, [ebp+80]
    mov eax, 70
    call GetString
    movb [edi+eax-1], 0xD
    movb [edi+eax], 0xA
    movb [edi+eax+1], 0

    orb [ebp+12], 8     # okay, echo now

    lea edx, [ebp]
    pop eax
    int 0x80            # ioctl TCSETS

    lea esi, crlf
    call PrintString
    call PrintString

    mov esi, edi

    lea edi, lencheckparams
    mov eax, [edi]
    mov ecx, [edi + 4]
    mov bl, 0x7F
    call CheckPass
    test eax, eax
    jnz sayFailed

    lea edi, passparams
    xor edx, edx
    mov dl, [passparamCount]
    dec edx

checkLoop:
    xor ebx, ebx
    mov eax, [edi + 8*edx]
    mov ecx, [edi + 8*edx + 4]
    call CheckPass
    
    test eax, eax
    jnz sayFailed
    dec edx
    jns checkLoop

    lea esi, correct
    call PrintString

    jmp doneSaying

sayFailed:
    lea esi, incorrect
    call PrintString
    mov bl, 0
    inc ebx

doneSaying:
    xor eax, eax
    inc eax
    int 0x80
# _start ENDP

# PrintString takes null-terminated string in ESI, prints it
PrintString:
    push eax
    push ebx
    push ecx
    push edx
    push esi
    xor eax, eax

sizeloop:
    cmpb [esi], 0
    jz endSizeLoop
    inc esi
    inc eax
    jmp sizeloop

endSizeLoop:
    pop esi
    mov edx, eax
    mov eax, 4
    mov ebx, 1
    mov ecx, esi
    int 0x80

    pop edx
    pop ecx
    pop ebx
    pop eax
    ret
# PrintString ENDP

# GetString gets a string from stdin into where EDI points, max EAX chars
# return length in EAX
GetString:
    push ebx
    push ecx
    push edx

    mov edx, eax
    mov eax, 3
    xor ebx, ebx
    mov ecx, edi
    int 0x80

    pop edx
    pop ecx
    pop ebx
    ret
# GetString ENDP

# tuning parameters in EAX and ECX, input mask in BL, null-terminated password in ESI
# retval in EAX
CheckPass:
    push ebp
    mov ebp, esp
    sub esp, 4
    movb [ebp-4], bl
    push esi
    push edx
    push ebx

cpStrLoop:
    cmpb [esi], 0
    jz endCpStrLoop
    mov bl, [esi]
    or bl, [ebp-4]
    mov dl, 8
    test bl, bl
    js  nonAscii         # require low bit-8 in password: ASCII only!

cpByteLoop0:
    rcl bl, 1
    rcl eax, 1
    jnc cpByteLoop1
    xor eax, ecx
cpByteLoop1:
    dec dl
    jnz cpByteLoop0

    inc esi
    jmp cpStrLoop

nonAscii:
    xor eax, eax
    dec eax
endCpStrLoop:
    pop ebx
    pop edx
    pop esi
    leave
    ret
# CheckPass ENDP