Analisis si un libro en excel esta abierto o no

Puedes hacerlo de esta manera, primero copia esto en el procedimiento general del formulario:
Private Const OPEN_EXISTING = 3
Private Type FILETIME
dwLowDateTime As Long
dwHighDateTime As Long
End Type
Private Type BY_HANDLE_FILE_INFORMATION
dwFileAttributes As Long
ftCreationTime As FILETIME
ftLastAccessTime As FILETIME
ftLastWriteTime As FILETIME
dwVolumeSerialNumber As Long
nFileSizeHigh As Long
nFileSizeLow As Long
nNumberOfLinks As Long
nFileIndexHigh As Long
nFileIndexLow As Long
End Type
Private Declare Function GetFileInformationByHandle Lib "kernel32" (ByVal hFile As Long, lpFileInformation As BY_HANDLE_FILE_INFORMATION) As Long
Private Declare Function CreateFile Lib "kernel32" Alias "CreateFileA" (ByVal lpFileName As String, ByVal dwDesiredAccess As Long, ByVal dwShareMode As Long, lpSecurityAttributes As Any, ByVal dwCreationDisposition As Long, ByVal dwFlagsAndAttributes As Long, ByVal hTemplateFile As Long) As Long
Private Declare Function CloseHandle Lib "kernel32" (ByVal hObject As Long) As Long
Function ArchivoAbierto(Archivo As String) As Boolean
Dim hFile As Long, FileInfo As BY_HANDLE_FILE_INFORMATION
hFile = CreateFile(Archivo, 0, 0, ByVal 0&, OPEN_EXISTING, 0, ByVal 0&)
GetFileInformationByHandle hFile, FileInfo
CloseHandle hFile
If CStr(FileInfo.nNumberOfLinks) = 0 Then
ArchivoAbierto = True
Else
ArchivoAbierto = False
End If
End Function
'ahora lo utilizas asi:
If ArchivoAbierto("c:\archivo.xls") = True Then
mmmmmmmm
Else
nnnnnnnn
End If
'esas funcion te entrega un boolena donde "True" significa archivo abierto, fijate de enviar el nombre y ruta del arhchivo correctamente.